[12/25] carbondata git commit: [CARBONDATA-2161] update mergeTo column for compacted segment of streaming table

[ravipesala]

[CARBONDATA-2161] update mergeTo column for compacted segment of streaming table

This closes #1971


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/36e770ce
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/36e770ce
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/36e770ce

Branch: refs/heads/branch-1.3
Commit: 36e770ce631e2bdb741fbd0d0ac60064399946f9
Parents: 95ac5ef
Author: BJangir 
Authored: Mon Feb 12 01:02:30 2018 +0530
Committer: ravipesala 
Committed: Sat Mar 3 17:55:40 2018 +0530

--
 .../org/apache/spark/carbondata/TestStreamingTableOperation.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/36e770ce/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 5644302..881af3a 100644
--- 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -1074,7 +1074,7 @@ class TestStreamingTableOperation extends QueryTest with 
BeforeAndAfterAll {
 //Verify MergeTO column entry for compacted Segments
 newSegments.filter(_.getString(1).equals("Compacted")).foreach{ rw =>
   assertResult("Compacted")(rw.getString(1))
-  assert(Integer.parseInt(rw.getString(0)) < 
Integer.parseInt(rw.getString(4)))
+  
assertResult((Integer.parseInt(rw.getString(0))+2).toString)(rw.getString(4))
 }
 checkAnswer(
   sql("select count(*) from streaming.stream_table_reopen"),

[01/16] carbondata git commit: [CARBONDATA-2161] update mergeTo column for compacted segment of streaming table

[gvramana]

Repository: carbondata
Updated Branches:
  refs/heads/branch-1.3 39ac94e46 -> 1997ca235


[CARBONDATA-2161] update mergeTo column for compacted segment of streaming table

This closes #1971


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c2785b35
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c2785b35
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c2785b35

Branch: refs/heads/branch-1.3
Commit: c2785b352f7b7cb2dd524811b0696fb18c12d5b0
Parents: 39ac94e
Author: BJangir 
Authored: Mon Feb 12 01:02:30 2018 +0530
Committer: Venkata Ramana G 
Committed: Tue Feb 27 12:45:55 2018 +0530

--
 .../apache/spark/carbondata/TestStreamingTableOperation.scala   | 5 +
 .../org/apache/carbondata/streaming/StreamHandoffRDD.scala  | 1 +
 2 files changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/c2785b35/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index a368cef..4b3a957 100644
--- 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -637,6 +637,11 @@ class TestStreamingTableOperation extends QueryTest with 
BeforeAndAfterAll {
 assertResult(newSegments.length / 
2)(newSegments.filter(_.getString(1).equals("Success")).length)
 assertResult(newSegments.length / 
2)(newSegments.filter(_.getString(1).equals("Compacted")).length)
 
+//Verify MergeTO column entry for compacted Segments
+newSegments.filter(_.getString(1).equals("Compacted")).foreach{ rw =>
+  assertResult("Compacted")(rw.getString(1))
+  
assertResult((Integer.parseInt(rw.getString(0))+2).toString)(rw.getString(4))
+}
 checkAnswer(
   sql("select count(*) from streaming.stream_table_reopen"),
   Seq(Row(2 * 100 * 2))

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c2785b35/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
--
diff --git 
a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
 
b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 41dfa50..4caa401 100644
--- 
a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ 
b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -398,6 +398,7 @@ object StreamHandoffRDD {
   throw new Exception("Failed to update table status for streaming 
segment")
 } else {
   streamSegment.get.setSegmentStatus(SegmentStatus.COMPACTED)
+  streamSegment.get.setMergedLoadName(loadModel.getSegmentId)
 }
 
 // refresh table status file

carbondata git commit: [CARBONDATA-2161] update mergeTo column for compacted segment of streaming table

[qiangcai]

Repository: carbondata
Updated Branches:
  refs/heads/master 001795ce2 -> 067b833f1


[CARBONDATA-2161] update mergeTo column for compacted segment of streaming table

This closes #1971


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/067b833f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/067b833f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/067b833f

Branch: refs/heads/master
Commit: 067b833f1f41bc4103a1d8248eb3d99e5990a127
Parents: 001795c
Author: BJangir 
Authored: Mon Feb 12 01:02:30 2018 +0530
Committer: QiangCai 
Committed: Fri Feb 23 14:36:24 2018 +0800

--
 .../apache/spark/carbondata/TestStreamingTableOperation.scala   | 5 +
 .../org/apache/carbondata/streaming/StreamHandoffRDD.scala  | 1 +
 2 files changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/067b833f/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index a368cef..4b3a957 100644
--- 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -637,6 +637,11 @@ class TestStreamingTableOperation extends QueryTest with 
BeforeAndAfterAll {
 assertResult(newSegments.length / 
2)(newSegments.filter(_.getString(1).equals("Success")).length)
 assertResult(newSegments.length / 
2)(newSegments.filter(_.getString(1).equals("Compacted")).length)
 
+//Verify MergeTO column entry for compacted Segments
+newSegments.filter(_.getString(1).equals("Compacted")).foreach{ rw =>
+  assertResult("Compacted")(rw.getString(1))
+  
assertResult((Integer.parseInt(rw.getString(0))+2).toString)(rw.getString(4))
+}
 checkAnswer(
   sql("select count(*) from streaming.stream_table_reopen"),
   Seq(Row(2 * 100 * 2))

http://git-wip-us.apache.org/repos/asf/carbondata/blob/067b833f/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
--
diff --git 
a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
 
b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 41dfa50..4caa401 100644
--- 
a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ 
b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -398,6 +398,7 @@ object StreamHandoffRDD {
   throw new Exception("Failed to update table status for streaming 
segment")
 } else {
   streamSegment.get.setSegmentStatus(SegmentStatus.COMPACTED)
+  streamSegment.get.setMergedLoadName(loadModel.getSegmentId)
 }
 
 // refresh table status file