[27/50] [abbrv] carbondata git commit: [CARBONDATA-2482] Pass uuid while writing segment file if possible

2018-07-17 Thread jackylk 
[CARBONDATA-2482] Pass uuid while writing segment file if possible

Pass the uuid (segmentFileName) to the writeSegmentFile method file if possible.

Problem:
When the supporting tables depends on the segmentFileName of the main table, 
query is failing, as it is expected to be the same name as that of the main 
table. Mostly in case of merge index, segmentFile will be rewritten for that 
segment.

Solution:
Whenever the supporting table segmentFileName should be same as that main 
table, in that case we should pass the name as the UUID in the merge index flow 
instead of taking a new timestamp.

This closes #2307


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6c5abddf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6c5abddf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6c5abddf

Branch: refs/heads/carbonstore
Commit: 6c5abddfb125b2dd51c9a989f003fe2bdc066d6d
Parents: eb604fd
Author: dhatchayani 
Authored: Tue May 15 12:24:01 2018 +0530
Committer: ravipesala 
Committed: Mon Jul 16 10:34:31 2018 +0530

--
 .../core/datamap/DataMapStoreManager.java   |  3 +-
 .../core/writer/CarbonIndexFileMergeWriter.java | 40 ++--
 .../sdv/generated/MergeIndexTestCase.scala  |  8 ++--
 .../CarbonIndexFileMergeTestCase.scala  | 18 -
 .../TestStreamingTableWithRowParser.scala   |  2 +-
 .../store/CarbonFactDataHandlerModel.java   |  3 +-
 .../processing/util/CarbonLoaderUtil.java   | 10 +++--
 7 files changed, 35 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c5abddf/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 8ce302b..475ec01 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -584,7 +584,8 @@ public final class DataMapStoreManager {
   SegmentRefreshInfo segmentRefreshInfo =
   seg.getSegmentRefreshInfo(updateVo);
   String segmentId = seg.getSegmentNo();
-  if (segmentRefreshTime.get(segmentId) == null) {
+  if (segmentRefreshTime.get(segmentId) == null
+  && segmentRefreshInfo.getSegmentUpdatedTimestamp() != null) {
 segmentRefreshTime.put(segmentId, segmentRefreshInfo);
 return true;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c5abddf/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
index 80a46cb..b080f52 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
@@ -67,7 +67,7 @@ public class CarbonIndexFileMergeWriter {
*/
   private String mergeCarbonIndexFilesOfSegment(String segmentId,
   String tablePath, List indexFileNamesTobeAdded,
-  boolean readFileFooterFromCarbonDataFile) throws IOException {
+  boolean readFileFooterFromCarbonDataFile, String uuid) throws 
IOException {
 Segment segment = Segment.getSegment(segmentId, tablePath);
 String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
 CarbonFile[] indexFiles;
@@ -85,7 +85,7 @@ public class CarbonIndexFileMergeWriter {
 readFileFooterFromCarbonDataFile, segmentPath, indexFiles, 
segmentId);
   } else {
 return writeMergeIndexFileBasedOnSegmentFile(
-segmentId, indexFileNamesTobeAdded, sfs, indexFiles);
+segmentId, indexFileNamesTobeAdded, sfs, indexFiles, uuid);
   }
 }
 return null;
@@ -111,10 +111,9 @@ public class CarbonIndexFileMergeWriter {
 return null;
   }
 
-  private String writeMergeIndexFileBasedOnSegmentFile(
-  String segmentId,
-  List indexFileNamesTobeAdded,
-  SegmentFileStore segmentFileStore, CarbonFile[] indexFiles) throws 
IOException {
+  private String writeMergeIndexFileBasedOnSegmentFile(String segmentId,
+  List indexFileNamesTobeAdded, SegmentFileStore segmentFileStore,
+  CarbonFile[] indexFiles, String uuid) throws IOException {
 SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
 fileStore

carbondata git commit: [CARBONDATA-2482] Pass uuid while writing segment file if possible

2018-07-15 Thread ravipesala 
Repository: carbondata
Updated Branches:
  refs/heads/master eb604fdb7 -> 6c5abddfb


[CARBONDATA-2482] Pass uuid while writing segment file if possible

Pass the uuid (segmentFileName) to the writeSegmentFile method file if possible.

Problem:
When the supporting tables depends on the segmentFileName of the main table, 
query is failing, as it is expected to be the same name as that of the main 
table. Mostly in case of merge index, segmentFile will be rewritten for that 
segment.

Solution:
Whenever the supporting table segmentFileName should be same as that main 
table, in that case we should pass the name as the UUID in the merge index flow 
instead of taking a new timestamp.

This closes #2307


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6c5abddf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6c5abddf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6c5abddf

Branch: refs/heads/master
Commit: 6c5abddfb125b2dd51c9a989f003fe2bdc066d6d
Parents: eb604fd
Author: dhatchayani 
Authored: Tue May 15 12:24:01 2018 +0530
Committer: ravipesala 
Committed: Mon Jul 16 10:34:31 2018 +0530

--
 .../core/datamap/DataMapStoreManager.java   |  3 +-
 .../core/writer/CarbonIndexFileMergeWriter.java | 40 ++--
 .../sdv/generated/MergeIndexTestCase.scala  |  8 ++--
 .../CarbonIndexFileMergeTestCase.scala  | 18 -
 .../TestStreamingTableWithRowParser.scala   |  2 +-
 .../store/CarbonFactDataHandlerModel.java   |  3 +-
 .../processing/util/CarbonLoaderUtil.java   | 10 +++--
 7 files changed, 35 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c5abddf/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 8ce302b..475ec01 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -584,7 +584,8 @@ public final class DataMapStoreManager {
   SegmentRefreshInfo segmentRefreshInfo =
   seg.getSegmentRefreshInfo(updateVo);
   String segmentId = seg.getSegmentNo();
-  if (segmentRefreshTime.get(segmentId) == null) {
+  if (segmentRefreshTime.get(segmentId) == null
+  && segmentRefreshInfo.getSegmentUpdatedTimestamp() != null) {
 segmentRefreshTime.put(segmentId, segmentRefreshInfo);
 return true;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c5abddf/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
index 80a46cb..b080f52 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
@@ -67,7 +67,7 @@ public class CarbonIndexFileMergeWriter {
*/
   private String mergeCarbonIndexFilesOfSegment(String segmentId,
   String tablePath, List indexFileNamesTobeAdded,
-  boolean readFileFooterFromCarbonDataFile) throws IOException {
+  boolean readFileFooterFromCarbonDataFile, String uuid) throws 
IOException {
 Segment segment = Segment.getSegment(segmentId, tablePath);
 String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
 CarbonFile[] indexFiles;
@@ -85,7 +85,7 @@ public class CarbonIndexFileMergeWriter {
 readFileFooterFromCarbonDataFile, segmentPath, indexFiles, 
segmentId);
   } else {
 return writeMergeIndexFileBasedOnSegmentFile(
-segmentId, indexFileNamesTobeAdded, sfs, indexFiles);
+segmentId, indexFileNamesTobeAdded, sfs, indexFiles, uuid);
   }
 }
 return null;
@@ -111,10 +111,9 @@ public class CarbonIndexFileMergeWriter {
 return null;
   }
 
-  private String writeMergeIndexFileBasedOnSegmentFile(
-  String segmentId,
-  List indexFileNamesTobeAdded,
-  SegmentFileStore segmentFileStore, CarbonFile[] indexFiles) throws 
IOException {
+  private String writeMergeIndexFileBasedOnSegmentFile(String segmentId,
+  List indexFileNamesTobeAdded, SegmentFileStore segmentFileStore,
+  CarbonFile[] indexFiles, String uuid) throws IOException {
 SegmentIndexFileStore fileStore = new