[12/25] carbondata git commit: [CARBONDATA-2161] update mergeTo column for compacted segment of streaming table
[CARBONDATA-2161] update mergeTo column for compacted segment of streaming table This closes #1971 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/36e770ce Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/36e770ce Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/36e770ce Branch: refs/heads/branch-1.3 Commit: 36e770ce631e2bdb741fbd0d0ac60064399946f9 Parents: 95ac5ef Author: BJangir Authored: Mon Feb 12 01:02:30 2018 +0530 Committer: ravipesala Committed: Sat Mar 3 17:55:40 2018 +0530 -- .../org/apache/spark/carbondata/TestStreamingTableOperation.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/36e770ce/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala -- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala index 5644302..881af3a 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala @@ -1074,7 +1074,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll { //Verify MergeTO column entry for compacted Segments newSegments.filter(_.getString(1).equals("Compacted")).foreach{ rw => assertResult("Compacted")(rw.getString(1)) - assert(Integer.parseInt(rw.getString(0)) < Integer.parseInt(rw.getString(4))) + assertResult((Integer.parseInt(rw.getString(0))+2).toString)(rw.getString(4)) } checkAnswer( sql("select count(*) from streaming.stream_table_reopen"),
[01/16] carbondata git commit: [CARBONDATA-2161] update mergeTo column for compacted segment of streaming table
Repository: carbondata Updated Branches: refs/heads/branch-1.3 39ac94e46 -> 1997ca235 [CARBONDATA-2161] update mergeTo column for compacted segment of streaming table This closes #1971 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c2785b35 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c2785b35 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c2785b35 Branch: refs/heads/branch-1.3 Commit: c2785b352f7b7cb2dd524811b0696fb18c12d5b0 Parents: 39ac94e Author: BJangir Authored: Mon Feb 12 01:02:30 2018 +0530 Committer: Venkata Ramana G Committed: Tue Feb 27 12:45:55 2018 +0530 -- .../apache/spark/carbondata/TestStreamingTableOperation.scala | 5 + .../org/apache/carbondata/streaming/StreamHandoffRDD.scala | 1 + 2 files changed, 6 insertions(+) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/c2785b35/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala -- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala index a368cef..4b3a957 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala @@ -637,6 +637,11 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll { assertResult(newSegments.length / 2)(newSegments.filter(_.getString(1).equals("Success")).length) assertResult(newSegments.length / 2)(newSegments.filter(_.getString(1).equals("Compacted")).length) +//Verify MergeTO column entry for compacted Segments +newSegments.filter(_.getString(1).equals("Compacted")).foreach{ rw => + assertResult("Compacted")(rw.getString(1)) + assertResult((Integer.parseInt(rw.getString(0))+2).toString)(rw.getString(4)) +} checkAnswer( sql("select count(*) from streaming.stream_table_reopen"), Seq(Row(2 * 100 * 2)) http://git-wip-us.apache.org/repos/asf/carbondata/blob/c2785b35/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala -- diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala index 41dfa50..4caa401 100644 --- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala +++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala @@ -398,6 +398,7 @@ object StreamHandoffRDD { throw new Exception("Failed to update table status for streaming segment") } else { streamSegment.get.setSegmentStatus(SegmentStatus.COMPACTED) + streamSegment.get.setMergedLoadName(loadModel.getSegmentId) } // refresh table status file
carbondata git commit: [CARBONDATA-2161] update mergeTo column for compacted segment of streaming table
Repository: carbondata Updated Branches: refs/heads/master 001795ce2 -> 067b833f1 [CARBONDATA-2161] update mergeTo column for compacted segment of streaming table This closes #1971 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/067b833f Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/067b833f Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/067b833f Branch: refs/heads/master Commit: 067b833f1f41bc4103a1d8248eb3d99e5990a127 Parents: 001795c Author: BJangir Authored: Mon Feb 12 01:02:30 2018 +0530 Committer: QiangCai Committed: Fri Feb 23 14:36:24 2018 +0800 -- .../apache/spark/carbondata/TestStreamingTableOperation.scala | 5 + .../org/apache/carbondata/streaming/StreamHandoffRDD.scala | 1 + 2 files changed, 6 insertions(+) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/067b833f/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala -- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala index a368cef..4b3a957 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala @@ -637,6 +637,11 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll { assertResult(newSegments.length / 2)(newSegments.filter(_.getString(1).equals("Success")).length) assertResult(newSegments.length / 2)(newSegments.filter(_.getString(1).equals("Compacted")).length) +//Verify MergeTO column entry for compacted Segments +newSegments.filter(_.getString(1).equals("Compacted")).foreach{ rw => + assertResult("Compacted")(rw.getString(1)) + assertResult((Integer.parseInt(rw.getString(0))+2).toString)(rw.getString(4)) +} checkAnswer( sql("select count(*) from streaming.stream_table_reopen"), Seq(Row(2 * 100 * 2)) http://git-wip-us.apache.org/repos/asf/carbondata/blob/067b833f/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala -- diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala index 41dfa50..4caa401 100644 --- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala +++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala @@ -398,6 +398,7 @@ object StreamHandoffRDD { throw new Exception("Failed to update table status for streaming segment") } else { streamSegment.get.setSegmentStatus(SegmentStatus.COMPACTED) + streamSegment.get.setMergedLoadName(loadModel.getSegmentId) } // refresh table status file