incubator-carbondata git commit: [maven-release-plugin] prepare for next development iteration

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/branch-1.1 d7bdfd54e -> d172bf5f4


[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d172bf5f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d172bf5f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d172bf5f

Branch: refs/heads/branch-1.1
Commit: d172bf5f47fbb2132183b4ed25abe31fa92fee86
Parents: d7bdfd5
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Tue Apr 18 18:08:33 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Tue Apr 18 18:08:33 2017 +0530

--
 assembly/pom.xml  | 2 +-
 common/pom.xml| 2 +-
 core/pom.xml  | 2 +-
 examples/flink/pom.xml| 2 +-
 examples/spark/pom.xml| 2 +-
 examples/spark2/pom.xml   | 2 +-
 format/pom.xml| 2 +-
 hadoop/pom.xml| 2 +-
 integration/spark-common-test/pom.xml | 2 +-
 integration/spark-common/pom.xml  | 2 +-
 integration/spark/pom.xml | 2 +-
 integration/spark2/pom.xml| 2 +-
 pom.xml   | 4 ++--
 processing/pom.xml| 2 +-
 14 files changed, 15 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d172bf5f/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 4d5f2dd..fdcc9b1 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d172bf5f/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 71561b0..56ad112 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d172bf5f/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index b589328..4ed5b1f 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d172bf5f/examples/flink/pom.xml
--
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index 5164315..4b7b4d9 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d172bf5f/examples/spark/pom.xml
--
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 10bdb93..6cb5547 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d172bf5f/examples/spark2/pom.xml
--
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index d28f606..df0a741 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d172bf5f/format/pom.xml
--
diff --git a/format/pom.xml b/format/pom.xml
index a47b45b..a0d69eb 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d172bf5f/hadoop/pom.xml
--
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index a31df27..7b840b9 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml

[incubator-carbondata] Git Push Summary

[ravipesala]

Repository: incubator-carbondata
Updated Tags:  refs/tags/apache-carbondata-1.1.0-incubating-rc2 [created] 
0ed699508

incubator-carbondata git commit: [maven-release-plugin] prepare release apache-carbondata-1.1.0-incubating-rc2

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/branch-1.1 dbf76485f -> d7bdfd54e


[maven-release-plugin] prepare release apache-carbondata-1.1.0-incubating-rc2


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d7bdfd54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d7bdfd54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d7bdfd54

Branch: refs/heads/branch-1.1
Commit: d7bdfd54e2c7c7ab0ebdb4f9f660bfd798bab610
Parents: dbf7648
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Tue Apr 18 18:07:13 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Tue Apr 18 18:07:13 2017 +0530

--
 assembly/pom.xml  | 2 +-
 common/pom.xml| 2 +-
 core/pom.xml  | 2 +-
 examples/flink/pom.xml| 2 +-
 examples/spark/pom.xml| 2 +-
 examples/spark2/pom.xml   | 2 +-
 format/pom.xml| 2 +-
 hadoop/pom.xml| 2 +-
 integration/spark-common-test/pom.xml | 2 +-
 integration/spark-common/pom.xml  | 2 +-
 integration/spark/pom.xml | 2 +-
 integration/spark2/pom.xml| 2 +-
 pom.xml   | 4 ++--
 processing/pom.xml| 2 +-
 14 files changed, 15 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d7bdfd54/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index c1b51f6..4d5f2dd 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d7bdfd54/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 91dcb1f..71561b0 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d7bdfd54/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index 976f8d2..b589328 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d7bdfd54/examples/flink/pom.xml
--
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index d3e8997..5164315 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d7bdfd54/examples/spark/pom.xml
--
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 3b65185..10bdb93 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d7bdfd54/examples/spark2/pom.xml
--
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index 10c59e7..d28f606 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d7bdfd54/format/pom.xml
--
diff --git a/format/pom.xml b/format/pom.xml
index c6c4b2a..a47b45b 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d7bdfd54/hadoop/pom.xml
--
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index e62fa20..a31df27 100644
--- a/hadoop/pom.xml
+++ b

[43/50] incubator-carbondata git commit: BigDecimal and VariableLength Dimension fixes

[ravipesala]

BigDecimal and VariableLength Dimension fixes


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/92352f3a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/92352f3a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/92352f3a

Branch: refs/heads/branch-1.1
Commit: 92352f3a695e99702564287921b57e72aa7f3a07
Parents: f58853d
Author: vpp9380 
Authored: Sat Apr 15 17:01:48 2017 +0530
Committer: Venkata Ramana G 
Committed: Sat Apr 15 22:04:01 2017 +0530

--
 .../store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java| 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/92352f3a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java
index 52bd74b..4082689 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java
@@ -124,11 +124,10 @@ public class UnsafeBigDecimalMeasureChunkStore extends 
UnsafeAbstractMeasureData
   int OffsetOfNextdata = 
CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
   dataPageMemoryBlock.getBaseOffset() + this.offsetStartPosition + 
((index + 1)
   * CarbonCommonConstants.INT_SIZE_IN_BYTE));
-  length =
-  (short) (OffsetOfNextdata - (currentDataOffset + 
CarbonCommonConstants.INT_SIZE_IN_BYTE));
+  length = OffsetOfNextdata - (currentDataOffset + 
CarbonCommonConstants.INT_SIZE_IN_BYTE);
 } else {
   // for last record we need to subtract with data length
-  length = (short) (this.offsetStartPosition - currentDataOffset);
+  length = (int) this.offsetStartPosition - currentDataOffset;
 }
 byte[] row = new byte[length];
 CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),

[39/50] incubator-carbondata git commit: Show segment link in data management doc is corrected show segment command output is reformatted

[ravipesala]

Show segment link in data management doc is corrected
show segment command output is reformatted

Fixed formatting


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f2f757d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f2f757d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f2f757d6

Branch: refs/heads/branch-1.1
Commit: f2f757d6b267284fa458eb2ebe49b1741e46d363
Parents: afa7887
Author: Sanoj MG 
Authored: Fri Apr 14 22:05:27 2017 +0400
Committer: chenliang613 
Committed: Sat Apr 15 16:36:21 2017 +0800

--
 docs/data-management.md | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f2f757d6/docs/data-management.md
--
diff --git a/docs/data-management.md b/docs/data-management.md
index 2663aff..297251d 100644
--- a/docs/data-management.md
+++ b/docs/data-management.md
@@ -47,12 +47,12 @@ This tutorial is going to introduce you to the conceptual 
details of data manage
 | Success | All the data is loaded into table and no bad records found. |
 | Partial Success | Data is loaded into table and bad records are found. Bad 
records are stored at carbon.badrecords.location. |

-   In case of failure, the error will be logged in error log. Details of loads 
can be seen with [SHOW SEGMENTS](dml-operation-on-carbondata.md) command. The 
show segment command output consists of :
+   In case of failure, the error will be logged in error log. Details of loads 
can be seen with [SHOW SEGMENTS](dml-operation-on-carbondata.md#show-segments) 
command. The show segment command output consists of :

-   - SegmentSequenceID
-   - START_TIME OF LOAD
-   - END_TIME OF LOAD 
-   - LOAD STATUS
+   - SegmentSequenceId
+   - Status
+   - Load Start Time
+   - Load End Time 
  
The latest load will be displayed first in the output.

[08/50] incubator-carbondata git commit: [CARBONDATA-901] Fix some spelling mistakes This closes #782

[ravipesala]

[CARBONDATA-901] Fix some spelling mistakes This closes #782


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/aea9ed60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/aea9ed60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/aea9ed60

Branch: refs/heads/branch-1.1
Commit: aea9ed608f75454267561df30324cd4b850fd32b
Parents: 58dc3aa 81ccec2
Author: chenliang613 
Authored: Wed Apr 12 00:44:28 2017 +0530
Committer: chenliang613 
Committed: Wed Apr 12 00:44:28 2017 +0530

--
 .../core/datastore/block/SegmentProperties.java | 24 ++--
 1 file changed, 12 insertions(+), 12 deletions(-)
--

[49/50] incubator-carbondata git commit: null value handled during compaction

[ravipesala]

null value handled during compaction


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8ed7931b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8ed7931b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8ed7931b

Branch: refs/heads/branch-1.1
Commit: 8ed7931bae6858a47c4989dde7223b20eecc7803
Parents: 4397d05
Author: rahulforallp 
Authored: Tue Apr 18 12:26:51 2017 +0530
Committer: rahulforallp 
Committed: Tue Apr 18 12:26:51 2017 +0530

--
 .../rowreader/AddColumnTestCases.scala  | 22 
 .../merger/CompactionResultSortProcessor.java   |  7 ---
 2 files changed, 26 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8ed7931b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
index e6fb265..06f480b 100644
--- 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
@@ -131,6 +131,28 @@ class AddColumnTestCases extends QueryTest with 
BeforeAndAfterAll {
 sql("DROP TABLE IF EXISTS carbon_table")
   }
 
+
+  test("test add column compaction") {
+sql("DROP TABLE IF EXISTS carbon_table")
+sql(
+  "CREATE TABLE carbon_table(intField int,stringField string,charField 
string,timestampField " +
+  "timestamp)STORED BY 'carbondata' TBLPROPERTIES" +
+  "('DICTIONARY_EXCLUDE'='charField')")
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO 
TABLE carbon_table " +
+
s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO 
TABLE carbon_table " +
+
s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO 
TABLE carbon_table " +
+
s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO 
TABLE carbon_table " +
+
s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+sql("Alter table carbon_table add columns(decimalField decimal(6,2))")
+
+sql("Alter table carbon_table compact 'minor'")
+
+sql("DROP TABLE IF EXISTS carbon_table")
+  }
+
   override def afterAll {
 sql("DROP TABLE IF EXISTS addcolumntest")
 sql("drop table if exists hivetable")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8ed7931b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
 
b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index ebf3683..8c1f577 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -184,8 +184,7 @@ public class CompactionResultSortProcessor extends 
AbstractResultProcessor {
*
* @param resultIteratorList
*/
-  private void processResult(List resultIteratorList)
-  throws Exception {
+  private void processResult(List resultIteratorList) 
throws Exception {
 for (RawResultIterator resultIterator : resultIteratorList) {
   while (resultIterator.hasNext()) {
 addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));
@@ -250,7 +249,9 @@ public class CompactionResultSortProcessor extends 
AbstractResultProcessor {
   private Object getConvertedMeasureValue(Object value, char aggType) {
 switch (aggType) {
   case CarbonCommonConstants.BIG_DECIMAL_MEASURE:
-value = ((org.apache.spark.sql.types.Decimal) 
value).toJavaBigDecimal();
+if (value != null) {
+  value = ((org.apache.spark.sql.types.Decimal) 
value).toJavaBigDecimal();
+}
 return value;
   default:
 return value;

[01/50] incubator-carbondata git commit: lazy rdd iterator [Forced Update!]

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/branch-1.1 8e50b878f -> dbf76485f (forced update)


lazy rdd iterator


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e52e6413
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e52e6413
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e52e6413

Branch: refs/heads/branch-1.1
Commit: e52e641372e298511ba0135054c97855153356dc
Parents: d51387b
Author: QiangCai 
Authored: Fri Apr 7 19:38:19 2017 +0530
Committer: Venkata Ramana G 
Committed: Tue Apr 11 14:08:53 2017 +0530

--
 .../spark/rdd/NewCarbonDataLoadRDD.scala| 100 +++
 1 file changed, 81 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e52e6413/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 72ee90f..a6d231d 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.{TaskAttemptID, TaskType}
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 import org.apache.spark.{Partition, SerializableWritable, SparkContext, 
SparkEnv, TaskContext}
 import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionWrap, RDD}
+import org.apache.spark.serializer.SerializerInstance
 import org.apache.spark.sql.Row
 import org.apache.spark.util.SparkUtil
 
@@ -408,19 +409,14 @@ class NewDataFrameLoaderRDD[K, V](
 val recordReaders = mutable.Buffer[CarbonIterator[Array[AnyRef]]]()
 val partitionIterator = 
firstParent[DataLoadPartitionWrap[Row]].iterator(theSplit, context)
 val serializer = SparkEnv.get.closureSerializer.newInstance()
-var serializeBuffer: ByteBuffer = null
+var serializeBytes: Array[Byte] = null
 while(partitionIterator.hasNext) {
   val value = partitionIterator.next()
-  val newInstance = {
-if (serializeBuffer == null) {
-  serializeBuffer = serializer.serialize[RDD[Row]](value.rdd)
-}
-serializeBuffer.rewind()
-serializer.deserialize[RDD[Row]](serializeBuffer)
+  if (serializeBytes == null) {
+serializeBytes = serializer.serialize[RDD[Row]](value.rdd).array()
   }
-  recordReaders += new 
NewRddIterator(newInstance.iterator(value.partition, context),
-  carbonLoadModel,
-  context)
+  recordReaders += new LazyRddIterator(serializer, serializeBytes, 
value.partition,
+  carbonLoadModel, context)
 }
 
 val loader = new SparkPartitionLoader(model,
@@ -477,15 +473,16 @@ class NewRddIterator(rddIter: Iterator[Row],
 carbonLoadModel: CarbonLoadModel,
 context: TaskContext) extends CarbonIterator[Array[AnyRef]] {
 
-  val timeStampformatString = 
CarbonProperties.getInstance().getProperty(CarbonCommonConstants
-.CARBON_TIMESTAMP_FORMAT, 
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
-  val timeStampFormat = new SimpleDateFormat(timeStampformatString)
-  val dateFormatString = 
CarbonProperties.getInstance().getProperty(CarbonCommonConstants
+  private val timeStampformatString = CarbonProperties.getInstance()
+.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+  CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+  private val timeStampFormat = new SimpleDateFormat(timeStampformatString)
+  private val dateFormatString = 
CarbonProperties.getInstance().getProperty(CarbonCommonConstants
 .CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
-  val dateFormat = new SimpleDateFormat(dateFormatString)
-  val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
-  val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
-  val serializationNullFormat =
+  private val dateFormat = new SimpleDateFormat(dateFormatString)
+  private val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
+  private val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
+  private val serializationNullFormat =
 
carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 
2)(1)
   def hasNext: Boolean = rddIter.hasNext
 
@@ -499,8 

[26/50] incubator-carbondata git commit: [CARBONDATA-883] Fixed cast exception for integer data types in RestructureBasedVectorResultCollector. This closes #771

[ravipesala]

[CARBONDATA-883] Fixed cast exception for integer data types in 
RestructureBasedVectorResultCollector. This closes #771


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/144620d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/144620d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/144620d9

Branch: refs/heads/branch-1.1
Commit: 144620d94845179169b54f587bf653db15ff8d60
Parents: 8478939 8c5c00c
Author: Venkata Ramana G 
Authored: Thu Apr 13 20:19:51 2017 +0530
Committer: Venkata Ramana G 
Committed: Thu Apr 13 20:19:51 2017 +0530

--
 .../RestructureBasedVectorResultCollector.java  | 28 ---
 .../scan/executor/util/RestructureUtil.java | 50 +++-
 .../scan/executor/util/RestructureUtilTest.java |  2 +-
 .../carbondata/spark/load/CarbonLoaderUtil.java |  2 +-
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |  6 +--
 .../execution/command/AlterTableCommands.scala  |  4 --
 .../apache/spark/sql/hive/CarbonMetastore.scala |  8 ++--
 .../vectorreader/AddColumnTestCases.scala   | 44 +
 8 files changed, 123 insertions(+), 21 deletions(-)
--

[19/50] incubator-carbondata git commit: [CARBONDATA-903] data load is not failing even though bad records exists in the data in case of unsafe sort or batch sort

[ravipesala]

[CARBONDATA-903] data load is not failing even though bad records exists in the 
data in case of unsafe sort or batch sort


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/53accb35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/53accb35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/53accb35

Branch: refs/heads/branch-1.1
Commit: 53accb35685fa959b5262a46518b6e9b0480439f
Parents: 9efcacd
Author: mohammadshahidkhan <mohdshahidkhan1...@gmail.com>
Authored: Tue Apr 11 18:26:51 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 16:07:58 2017 +0530

--
 .../DataLoadFailAllTypeSortTest.scala   | 218 +++
 .../newflow/sort/AbstractMergeSorter.java   |  43 
 .../sort/impl/ParallelReadMergeSorterImpl.java  |  18 +-
 ...arallelReadMergeSorterWithBucketingImpl.java |  16 +-
 .../UnsafeBatchParallelReadMergeSorterImpl.java |  43 +++-
 .../impl/UnsafeParallelReadMergeSorterImpl.java |  19 +-
 .../UnsafeSingleThreadFinalSortFilesMerger.java |  10 +
 7 files changed, 333 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/53accb35/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala
new file mode 100644
index 000..478b4d3
--- /dev/null
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.carbondata
+
+import java.io.File
+
+import org.apache.spark.sql.common.util.QueryTest
+import org.apache.spark.sql.hive.HiveContext
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * Test Class for detailed query on timestamp datatypes
+ *
+ *
+ */
+class DataLoadFailAllTypeSortTest extends QueryTest with BeforeAndAfterAll {
+  var hiveContext: HiveContext = _
+
+  override def beforeAll: Unit = {
+sql("drop table IF EXISTS data_pm")
+sql("drop table IF EXISTS data_um")
+sql("drop table IF EXISTS data_bm")
+sql("drop table IF EXISTS data_bmf")
+sql("drop table IF EXISTS data_tbm")
+  }
+
+  test("dataload with parallel merge with bad_records_action='FAIL'") {
+try {
+  CarbonProperties.getInstance()
+.addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+  new File("./target/test/badRecords")
+.getCanonicalPath)
+  CarbonProperties.getInstance()
+.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, 
"/MM/dd")
+  CarbonProperties.getInstance()
+.addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FAIL");
+  sql("create table data_pm(name String, dob long, weight int) " +
+  "STORED BY 'org.apache.carbondata.format'")
+  val testData = s"$resourcesPath/badrecords/dummy.csv"
+  sql(s"""LOAD DATA LOCAL INPATH '$testData' INTO table data_pm""")
+
+
+} catch {
+  case x: Throwable => {
+assert(x.getMessage.contains("Data load failed due to bad record"))
+CarbonProperties.getInstance()
+  .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, 
"dd-MM-")
+  }
+}
+finally {
+  CarbonProperties.getInstance()
+.addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+  CarbonCommonConstants.

[30/50] incubator-carbondata git commit: [CARBONDATA-900] Is null query on a newly added measure column is not returning proper results. This closes #781

[ravipesala]

[CARBONDATA-900] Is null query on a newly added measure column is not returning 
proper results. This closes #781


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/27a13843
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/27a13843
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/27a13843

Branch: refs/heads/branch-1.1
Commit: 27a1384385f5e857677bbe0b7ef0663851125efc
Parents: 51f9629 1e8d26c
Author: Venkata Ramana G 
Authored: Fri Apr 14 11:45:42 2017 +0530
Committer: Venkata Ramana G 
Committed: Fri Apr 14 11:45:42 2017 +0530

--
 .../carbondata/core/scan/filter/FilterUtil.java | 10 +++--
 .../executer/RowLevelFilterExecuterImpl.java|  8 
 .../core/scan/filter/FilterUtilTest.java| 12 +
 .../src/test/resources/restructure/data6.csv|  3 ++
 .../vectorreader/AddColumnTestCases.scala   | 46 
 5 files changed, 75 insertions(+), 4 deletions(-)
--

[46/50] incubator-carbondata git commit: [CARBONDATA-932] Fixed variable length filter query with empty data. This closes #804

[ravipesala]

[CARBONDATA-932] Fixed variable length filter query with empty data. This 
closes #804


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a8ed450b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a8ed450b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a8ed450b

Branch: refs/heads/branch-1.1
Commit: a8ed450bf1546dd69891c26de4661e18008ac11c
Parents: c73e02f 0729379
Author: Venkata Ramana G 
Authored: Sat Apr 15 22:22:40 2017 +0530
Committer: Venkata Ramana G 
Committed: Sat Apr 15 22:22:40 2017 +0530

--
 .../UnsafeVariableLengthDimesionDataChunkStore.java   | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--

[38/50] incubator-carbondata git commit: [CARBONDATA-911] Updated ddl-operation-on-carbondata.md This closes #787

[ravipesala]

[CARBONDATA-911] Updated ddl-operation-on-carbondata.md This closes #787


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/afa7887b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/afa7887b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/afa7887b

Branch: refs/heads/branch-1.1
Commit: afa7887b853f150e81d75acca446a859feffc8d5
Parents: 69ba223 ad73854
Author: chenliang613 
Authored: Sat Apr 15 16:29:40 2017 +0800
Committer: chenliang613 
Committed: Sat Apr 15 16:29:40 2017 +0800

--
 docs/ddl-operation-on-carbondata.md | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--

[33/50] incubator-carbondata git commit: added AlterTableAddColumnRDD to AlterTableCommands

[ravipesala]

added AlterTableAddColumnRDD to AlterTableCommands


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/36112fa4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/36112fa4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/36112fa4

Branch: refs/heads/branch-1.1
Commit: 36112fa46d2a15e6403af97a7d8677231bedc8d0
Parents: f27b491
Author: kunal642 
Authored: Tue Apr 11 14:15:10 2017 +0530
Committer: Venkata Ramana G 
Committed: Fri Apr 14 16:16:52 2017 +0530

--
 .../spark/rdd/AlterTableAddColumnRDD.scala  |  6 +-
 .../execution/command/carbonTableSchema.scala   |  8 +--
 .../execution/command/AlterTableCommands.scala  | 62 
 .../org/apache/spark/util/AlterTableUtil.scala  | 16 +++--
 .../restructure/AlterTableRevertTestCase.scala  | 19 +-
 5 files changed, 68 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/36112fa4/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
index bb65b0b..ab1fd9c 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
@@ -19,7 +19,6 @@ package org.apache.carbondata.spark.rdd
 
 import org.apache.spark.{Partition, SparkContext, TaskContext}
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.execution.command.AlterTableAddColumnsModel
 
 import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -49,13 +48,12 @@ class AddColumnPartition(rddId: Int, idx: Int, schema: 
ColumnSchema) extends Par
  */
 class AlterTableAddColumnRDD[K, V](sc: SparkContext,
 @transient newColumns: Seq[ColumnSchema],
-alterTableModel: AlterTableAddColumnsModel,
 carbonTableIdentifier: CarbonTableIdentifier,
 carbonStorePath: String) extends RDD[(Int, String)](sc, Nil) {
 
   override def getPartitions: Array[Partition] = {
 newColumns.zipWithIndex.map { column =>
-  new DropColumnPartition(id, column._2, column._1)
+  new AddColumnPartition(id, column._2, column._1)
 }.toArray
   }
 
@@ -65,7 +63,7 @@ class AlterTableAddColumnRDD[K, V](sc: SparkContext,
 val status = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
 val iter = new Iterator[(Int, String)] {
   try {
-val columnSchema = split.asInstanceOf[DropColumnPartition].columnSchema
+val columnSchema = split.asInstanceOf[AddColumnPartition].columnSchema
 // create dictionary file if it is a dictionary column
 if (columnSchema.hasEncoding(Encoding.DICTIONARY) &&
 !columnSchema.hasEncoding(Encoding.DIRECT_DICTIONARY)) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/36112fa4/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 117b365..5108df8 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -152,7 +152,7 @@ case class AlterTableDropColumnModel(databaseName: 
Option[String],
 tableName: String,
 columns: List[String])
 
-class AlterTableProcessor(
+class AlterTableColumnSchemaGenerator(
 alterTableModel: AlterTableAddColumnsModel,
 dbName: String,
 tableInfo: TableInfo,
@@ -253,12 +253,6 @@ class AlterTableProcessor(
 }
   }
 }
-// generate dictionary files for the newly added columns
-new AlterTableAddColumnRDD(sc,
-  newCols,
-  alterTableModel,
-  tableIdentifier,
-  storePath).collect()
 tableSchema.setListOfColumns(allColumns.asJava)
 tableInfo.setLastUpdatedTime(System.currentTimeMillis())
 tableInfo.setFactTable(tableSchema)

[27/50] incubator-carbondata git commit: Problem: After drop table dictionary and BTree instances are not getting cleared from driver memory. Due to this memory will keep growing and after some time G

[ravipesala]

Problem: After drop table dictionary and BTree instances are not getting 
cleared from driver memory. Due to this memory will keep growing and after some 
time GC problems will occur. In real case scenarios usually driver memory is on 
lower side hence it is more prone to GC problems.

Solution:
1. When a table is being clear BTree and dictionary instances from LRU cache.
2. Clear the access count for each segment immediately after block pruning 
rather then loading all the segments first and at lats clearing access count 
for all the segments together.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/673f8c22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/673f8c22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/673f8c22

Branch: refs/heads/branch-1.1
Commit: 673f8c2263c94e3475552de4da06327029f06eb3
Parents: 144620d
Author: manishgupta88 
Authored: Tue Apr 11 12:21:35 2017 +0530
Committer: manishgupta88 
Committed: Thu Apr 13 21:13:08 2017 +0530

--
 .../core/cache/dictionary/ManageDictionary.java | 108 -
 .../dictionary/ManageDictionaryAndBTree.java| 158 +++
 .../carbondata/hadoop/CacheAccessClient.java|  23 ++-
 .../carbondata/hadoop/CarbonInputFormat.java| 153 +-
 .../spark/rdd/AlterTableDropColumnRDD.scala |   4 +-
 .../execution/command/carbonTableSchema.scala   |   8 +-
 .../carbondata/CarbonDataSourceSuite.scala  |   2 +-
 7 files changed, 269 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/673f8c22/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionary.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionary.java
 
b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionary.java
deleted file mode 100644
index 0a38890..000
--- 
a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionary.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.cache.dictionary;
-
-import java.io.IOException;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.ColumnIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-
-/**
- * This class is aimed at managing dictionary files for any new addition and 
deletion
- * and calling of clear cache for the non existing dictionary files
- */
-public class ManageDictionary {
-
-  /**
-   * Attribute for Carbon LOGGER
-   */
-  private static final LogService LOGGER =
-  LogServiceFactory.getLogService(ManageDictionary.class.getName());
-
-  /**
-   * This method will delete the dictionary files for the given column IDs and
-   * clear the dictionary cache
-   *
-   * @param columnSchema
-   * @param carbonTableIdentifier
-   * @param storePath
-   */
-  public static void deleteDictionaryFileAndCache(final ColumnSchema 
columnSchema,
-  CarbonTableIdentifier carbonTableIdentifier, String storePath) {
-CarbonTablePath carbonTablePath =
-CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier);
-String 

[25/50] incubator-carbondata git commit: fixed cast exception for integer data types in RestructureBasedVectorResultCollector

[ravipesala]

fixed cast exception for integer data types in 
RestructureBasedVectorResultCollector

fixed cast exception for double datatype


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8c5c00c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8c5c00c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8c5c00c0

Branch: refs/heads/branch-1.1
Commit: 8c5c00c0c169d74adb72f13b8a781c893a6ad791
Parents: 8478939
Author: kunal642 
Authored: Fri Apr 7 18:56:16 2017 +0530
Committer: Venkata Ramana G 
Committed: Thu Apr 13 19:15:54 2017 +0530

--
 .../RestructureBasedVectorResultCollector.java  | 28 ---
 .../scan/executor/util/RestructureUtil.java | 50 +++-
 .../scan/executor/util/RestructureUtilTest.java |  2 +-
 .../carbondata/spark/load/CarbonLoaderUtil.java |  2 +-
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |  6 +--
 .../execution/command/AlterTableCommands.scala  |  4 --
 .../apache/spark/sql/hive/CarbonMetastore.scala |  8 ++--
 .../vectorreader/AddColumnTestCases.scala   | 44 +
 8 files changed, 123 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8c5c00c0/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
index 99b7f01..c09ec12 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
@@ -16,7 +16,6 @@
  */
 package org.apache.carbondata.core.scan.collector.impl;
 
-import java.math.BigDecimal;
 import java.util.List;
 
 import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -36,10 +35,13 @@ import org.apache.spark.sql.types.Decimal;
  */
 public class RestructureBasedVectorResultCollector extends 
DictionaryBasedVectorResultCollector {
 
+  private Object[] measureDefaultValues = null;
+
   public RestructureBasedVectorResultCollector(BlockExecutionInfo 
blockExecutionInfos) {
 super(blockExecutionInfos);
 queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
 queryMeasures = tableBlockExecutionInfos.getActualQueryMeasures();
+measureDefaultValues = new Object[queryMeasures.length];
 allColumnInfo = new ColumnVectorInfo[queryDimensions.length + 
queryMeasures.length];
 createVectorForNewlyAddedDimensions();
 createVectorForNewlyAddedMeasures();
@@ -68,10 +70,23 @@ public class RestructureBasedVectorResultCollector extends 
DictionaryBasedVector
 // add a dummy column vector result collector object
 ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
 allColumnInfo[queryMeasures[i].getQueryOrder()] = columnVectorInfo;
+measureDefaultValues[i] = 
getMeasureDefaultValue(queryMeasures[i].getMeasure());
   }
 }
   }
 
+  /**
+   * Gets the default value for each CarbonMeasure
+   * @param carbonMeasure
+   * @return
+   */
+  private Object getMeasureDefaultValue(CarbonMeasure carbonMeasure) {
+return 
RestructureUtil.getMeasureDefaultValueByType(carbonMeasure.getColumnSchema(),
+carbonMeasure.getDefaultValue());
+  }
+
+
+
   @Override public List collectData(AbstractScannedResult 
scannedResult, int batchSize) {
 throw new UnsupportedOperationException("collectData is not supported 
here");
   }
@@ -177,11 +192,11 @@ public class RestructureBasedVectorResultCollector 
extends DictionaryBasedVector
   private void fillDataForNonExistingMeasures() {
 for (int i = 0; i < 
tableBlockExecutionInfos.getActualQueryMeasures().length; i++) {
   if (!measureInfo.getMeasureExists()[i]) {
+int queryOrder = 
tableBlockExecutionInfos.getActualQueryMeasures()[i].getQueryOrder();
 CarbonMeasure measure = 
tableBlockExecutionInfos.getActualQueryMeasures()[i].getMeasure();
-ColumnVectorInfo columnVectorInfo = allColumnInfo[i];
-CarbonColumnVector vector = allColumnInfo[i].vector;
-Object defaultValue = RestructureUtil
-.getMeasureDefaultValue(measure.getColumnSchema(), 
measure.getDefaultValue());
+ColumnVectorInfo columnVectorInfo = allColumnInfo[queryOrder];
+CarbonColumnVector vector = columnVectorInfo.vector;
+Object defaultValue = measureDefaultValues[i];

[31/50] incubator-carbondata git commit: fixed NullPointerException for 2nd level subquery

[ravipesala]

fixed NullPointerException for 2nd level subquery


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/fc0fc7d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/fc0fc7d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/fc0fc7d9

Branch: refs/heads/branch-1.1
Commit: fc0fc7d906c9f6579bc3fc67b5672cac4de55068
Parents: 27a1384
Author: kunal642 
Authored: Tue Apr 11 16:43:39 2017 +0530
Committer: Venkata Ramana G 
Committed: Fri Apr 14 12:54:09 2017 +0530

--
 .../src/test/resources/temp/data1.csv   |  4 ++
 .../sql/optimizer/CarbonLateDecodeRule.scala|  6 ++-
 .../carbondata/query/SubQueryTestSuite.scala| 44 
 3 files changed, 53 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc0fc7d9/integration/spark-common-test/src/test/resources/temp/data1.csv
--
diff --git a/integration/spark-common-test/src/test/resources/temp/data1.csv 
b/integration/spark-common-test/src/test/resources/temp/data1.csv
new file mode 100644
index 000..44e9c10
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/temp/data1.csv
@@ -0,0 +1,4 @@
+id,name,rating
+1,xyz,5
+2,ghj,2
+3,ghj,3

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc0fc7d9/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
index 181328d..45cc330 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
@@ -60,7 +60,11 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with 
PredicateHelper {
 if (relations.nonEmpty && !isOptimized(plan)) {
   // In case scalar subquery skip the transformation and update the flag.
   if (relations.exists(_.carbonRelation.isSubquery.nonEmpty)) {
-relations.foreach(p => p.carbonRelation.isSubquery.remove(0))
+relations.foreach{carbonDecoderRelation =>
+  if (carbonDecoderRelation.carbonRelation.isSubquery.nonEmpty) {
+carbonDecoderRelation.carbonRelation.isSubquery.remove(0)
+  }
+}
 LOGGER.info("Skip CarbonOptimizer for scalar/predicate sub query")
 return plan
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc0fc7d9/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/SubQueryTestSuite.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/SubQueryTestSuite.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/SubQueryTestSuite.scala
new file mode 100644
index 000..fbc859f
--- /dev/null
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/SubQueryTestSuite.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.carbondata.query
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class SubQueryTestSuite extends QueryTest with BeforeAndAfterAll {
+
+  val tempDirPath = s"$resourcesPath/temp"
+
+  override def beforeAll(){
+sql("drop table if exists subquery")
+sql("create table subquery(id int, name string, rating float) stored by 
'carbondata'")
+sql(s"load data local inpath '$tempDirPath/data1.csv' into table subquery")
+  }
+
+  test("test to check if 2nd level subquery gives correct result") {
+

[11/50] incubator-carbondata git commit: [CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured with the executor lru cache size.

[ravipesala]

[CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured 
with the executor lru cache size.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/99582a76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/99582a76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/99582a76

Branch: refs/heads/branch-1.1
Commit: 99582a762871c8a2c6cc9f5db43718d21e13de0c
Parents: 3966f99
Author: mohammadshahidkhan <mohdshahidkhan1...@gmail.com>
Authored: Tue Apr 11 09:59:45 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 12 14:10:34 2017 +0530

--
 .../core/cache/CacheProviderTest.java   | 53 
 .../scala/org/apache/spark/sql/CarbonEnv.scala  |  1 +
 2 files changed, 54 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/99582a76/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java 
b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
index 1be5cb2..4c12259 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
@@ -17,16 +17,26 @@
 
 package org.apache.carbondata.core.cache;
 
+import java.io.IOException;
+import java.lang.reflect.Field;
+
 import org.apache.carbondata.core.cache.dictionary.Dictionary;
 import 
org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
 import org.apache.carbondata.core.cache.dictionary.ForwardDictionaryCache;
 import org.apache.carbondata.core.cache.dictionary.ReverseDictionaryCache;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.BlockIndexStore;
+import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
+import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
+import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
 import org.apache.carbondata.core.util.CarbonProperties;
 
+import org.apache.avro.Schema;
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -39,6 +49,9 @@ public class CacheProviderTest {
 // enable lru cache by setting cache size
 CarbonProperties.getInstance()
 .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, 
"10");
+// enable lru cache by setting cache size
+CarbonProperties.getInstance()
+.addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, 
"20");
   }
 
   @Test public void getInstance() throws Exception {
@@ -62,4 +75,44 @@ public class CacheProviderTest {
 assertTrue(reverseDictionaryCache instanceof ReverseDictionaryCache);
 assertFalse(reverseDictionaryCache instanceof ForwardDictionaryCache);
   }
+
+  /**
+   * to test the driver and executor lru memory configuration
+   *
+   * @throws IOException
+   * @throws NoSuchFieldException
+   * @throws IllegalAccessException
+   */
+  @Test public void driverExecutorCacheConfTest()
+  throws IOException, NoSuchFieldException, IllegalAccessException {
+// get cache provider instance
+CacheProvider cacheProvider = CacheProvider.getInstance();
+
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE,
 "true");
+Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexStore> driverCache =
+cacheProvider.createCache(CacheType.DRIVER_BTREE, "carbonStore");
+Field carbonLRUCacheField = 
SegmentTaskIndexStore.class.getDeclaredField("lruCache");
+carbonLRUCacheField.setAccessible(true);
+CarbonLRUCache carbonLRUCache = (CarbonLRUCache) 
carbonLRUCacheField.get(driverCache);
+Field lruCacheMemorySizeField = 
CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize");
+lruCacheMemorySizeField.setAccessible(true);
+long lruCacheMemorySize = (long) 
lruCacheMemorySizeField.get(carbonLRUCache);
+String driverCacheSize = CarbonProperties.getInstance()
+.getProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE);
+assertEquals(1024 * 1024 * Integer.parseInt(driverCacheSize), 
lruCacheMemorySize);
+// drop cache
+cacheProvider.dropAllCache();
+// validation test for the executor memory.
+
CarbonProperties.getInstance

[35/50] incubator-carbondata git commit: Added link to configuration parameters in main page of doc

[ravipesala]

Added link to configuration parameters in main page of doc


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e1b1713f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e1b1713f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e1b1713f

Branch: refs/heads/branch-1.1
Commit: e1b1713f9c08395e8818b4569bff95633fe8d841
Parents: 19b9223
Author: Sanoj MG 
Authored: Fri Apr 14 22:35:23 2017 +0400
Committer: Sanoj MG 
Committed: Fri Apr 14 22:35:23 2017 +0400

--
 README.md | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e1b1713f/README.md
--
diff --git a/README.md b/README.md
index db1a1e2..fafbf7a 100644
--- a/README.md
+++ b/README.md
@@ -46,6 +46,7 @@ CarbonData is built using Apache Maven, to [build 
CarbonData](https://github.com
 * [DDL Operations on 
CarbonData](https://github.com/apache/incubator-carbondata/blob/master/docs/ddl-operation-on-carbondata.md)
 
 * [DML Operations on 
CarbonData](https://github.com/apache/incubator-carbondata/blob/master/docs/dml-operation-on-carbondata.md)
  
 * [Cluster Installation and 
Deployment](https://github.com/apache/incubator-carbondata/blob/master/docs/installation-guide.md)
+* [Configuring 
Carbondata](https://github.com/apache/incubator-carbondata/blob/master/docs/configuration-parameters.md)
 * [FAQ](https://github.com/apache/incubator-carbondata/blob/master/docs/faq.md)
 * [Trouble 
Shooting](https://github.com/apache/incubator-carbondata/blob/master/docs/troubleshooting.md)
 * [Useful 
Tips](https://github.com/apache/incubator-carbondata/blob/master/docs/useful-tips-on-carbondata.md)

[17/50] incubator-carbondata git commit: Updating proper schema update time stamp

[ravipesala]

Updating proper schema update time stamp


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ebc5ee97
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ebc5ee97
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ebc5ee97

Branch: refs/heads/branch-1.1
Commit: ebc5ee973cf6e55194d9804a7a70b2011505339a
Parents: 11df451
Author: Manohar <manohar.craz...@gmail.com>
Authored: Wed Apr 12 16:22:26 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 15:36:19 2017 +0530

--
 .../MajorCompactionIgnoreInMinorTest.scala  | 53 +---
 .../spark/rdd/CarbonDataRDDFactory.scala| 11 ++--
 .../store/CarbonFactDataHandlerModel.java   |  1 +
 3 files changed, 54 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ebc5ee97/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index a879598..9d2cf96 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -35,6 +35,10 @@ import org.apache.carbondata.hadoop.CacheClient
   */
 class MajorCompactionIgnoreInMinorTest extends QueryTest with 
BeforeAndAfterAll {
 
+  val csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
+  val csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
+  val csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
+
   override def beforeAll {
 
CarbonProperties.getInstance().addProperty("carbon.compaction.level.threshold", 
"2,2")
 sql("drop table if exists  ignoremajor")
@@ -46,12 +50,6 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest 
with BeforeAndAfterAll
 "phonetype String, serialname String, salary Int) STORED BY 
'org.apache.carbondata" +
 ".format'"
 )
-
-
-val csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
-val csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
-val csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
-
 sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE ignoremajor 
OPTIONS" +
   "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
 )
@@ -149,8 +147,51 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest 
with BeforeAndAfterAll
 
   }
 
+  /**
+   * Execute two major compactions sequentially
+   */
+  test("Execute two major compactions sequentially") {
+sql("drop table if exists testmajor")
+sql(
+  "CREATE TABLE IF NOT EXISTS testmajor (country String, ID Int, date 
Timestamp, name " +
+  "String, " +
+  "phonetype String, serialname String, salary Int) STORED BY 
'org.apache.carbondata" +
+  ".format'"
+)
+sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE testmajor 
OPTIONS" +
+"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+)
+sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE testmajor  
OPTIONS" +
+"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+)
+// compaction will happen here.
+sql("alter table testmajor compact 'major'")
+sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE testmajor 
OPTIONS" +
+"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+)
+sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE testmajor  
OPTIONS" +
+"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+)
+sql("alter table testmajor compact 'major'")
+val identifier = new AbsoluteTableIdentifier(
+  
CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
+  new CarbonTableIdentifier(
+CarbonCommonConstants.DATABASE_DEFAULT_NAME, "testmajor", "ttt")
+)
+val segmentStatusManager: SegmentStatusManager = new 
SegmentStatusManager(identifi

[03/50] incubator-carbondata git commit: Removed unnecessary array copy and bitset checking

[ravipesala]

Removed unnecessary array copy and bitset checking

OPtimized code

Added table_blocksize option.

Removed unnecessary plan from optimized plan.

Fixed test

FIxed comment

Rebased


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8f59a326
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8f59a326
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8f59a326

Branch: refs/heads/branch-1.1
Commit: 8f59a326ea4028ff7987e98826b00926df804ea7
Parents: 00535f4
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Sun Mar 5 20:32:35 2017 +0530
Committer: jackylk <jacky.li...@huawei.com>
Committed: Tue Apr 11 21:01:53 2017 +0800

--
 .../carbondata/core/datastore/FileHolder.java   |   4 +-
 .../impl/VariableLengthDimensionDataChunk.java  |  16 +-
 ...mpressedDimensionChunkFileBasedReaderV1.java |  14 +-
 ...mpressedDimensionChunkFileBasedReaderV2.java |  19 +--
 ...mpressedDimensionChunkFileBasedReaderV3.java |  16 +-
 ...CompressedMeasureChunkFileBasedReaderV1.java |  16 +-
 ...CompressedMeasureChunkFileBasedReaderV2.java |  19 +--
 ...CompressedMeasureChunkFileBasedReaderV3.java |  34 ++--
 .../chunk/store/DimensionDataChunkStore.java|   9 ++
 .../SafeAbsractDimensionDataChunkStore.java |   4 +
 ...feVariableLengthDimensionDataChunkStore.java |  36 -
 .../UnsafeAbstractDimensionDataChunkStore.java  |   4 +
 ...afeVariableLengthDimesionDataChunkStore.java |  12 ++
 .../core/datastore/impl/DFSFileHolderImpl.java  |   8 +-
 .../core/datastore/impl/FileHolderImpl.java |   8 +-
 .../DictionaryBasedVectorResultCollector.java   |   1 -
 .../executor/impl/AbstractQueryExecutor.java|   2 +-
 .../core/scan/executor/util/QueryUtil.java  |   2 +-
 .../vector/MeasureDataVectorProcessor.java  | 158 +--
 .../apache/carbondata/core/util/ByteUtil.java   |  29 
 .../apache/carbondata/core/util/CarbonUtil.java |  12 +-
 .../carbondata/examples/CompareTest.scala   |   4 +
 .../apache/carbondata/spark/CarbonOption.scala  |   2 +
 .../VectorizedCarbonRecordReader.java   |   2 +-
 .../spark/sql/CarbonCatalystOperators.scala |   2 +-
 .../spark/sql/CarbonDataFrameWriter.scala   |  13 +-
 .../spark/sql/CarbonDictionaryDecoder.scala |  68 
 .../execution/CarbonLateDecodeStrategy.scala|   5 +-
 28 files changed, 328 insertions(+), 191 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8f59a326/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java 
b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
index b1eb1ee..1b972bc 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
@@ -27,12 +27,12 @@ public interface FileHolder {
* and length(number of bytes) need to read
*
* @param filePath fully qualified file path
-   * @param byteBuffer
* @param offset reading start position,
* @param length number of bytes to be read
+   * @return ByteBuffer
* @throws IOException
*/
-  void readByteBuffer(String filePath, ByteBuffer byteBuffer, long offset, int 
length)
+  ByteBuffer readByteBuffer(String filePath, long offset, int length)
   throws IOException;
   /**
* This method will be used to read the byte array from file based on offset

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8f59a326/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index eac06f4..6c47bf5 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -16,8 +16,6 @@
  */
 package org.apache.carbondata.core.datastore.chunk.impl;
 
-import java.util.Arrays;
-
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import 
org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
 import 
org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
@@ -109,14 +107,9 @@ public class VariableLengthDimensionDataChunk extends 
AbstractDimensionDataCh

[20/50] incubator-carbondata git commit: [CARBONDATA-903] data load is not failing even though bad records exists in the data in case of unsafe sort or batch sort. This closes #783

[ravipesala]

[CARBONDATA-903] data load is not failing even though bad records exists in the 
data in case of unsafe sort or batch sort. This closes #783


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f4fc6519
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f4fc6519
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f4fc6519

Branch: refs/heads/branch-1.1
Commit: f4fc651990d5f3865ef93dd6fb3ac49920cff0e5
Parents: 9efcacd 53accb3
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 13 16:08:43 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 16:08:43 2017 +0530

--
 .../DataLoadFailAllTypeSortTest.scala   | 218 +++
 .../newflow/sort/AbstractMergeSorter.java   |  43 
 .../sort/impl/ParallelReadMergeSorterImpl.java  |  18 +-
 ...arallelReadMergeSorterWithBucketingImpl.java |  16 +-
 .../UnsafeBatchParallelReadMergeSorterImpl.java |  43 +++-
 .../impl/UnsafeParallelReadMergeSorterImpl.java |  19 +-
 .../UnsafeSingleThreadFinalSortFilesMerger.java |  10 +
 7 files changed, 333 insertions(+), 34 deletions(-)
--

[41/50] incubator-carbondata git commit: Drop table IS is throwing exception

[ravipesala]

Drop table IS is throwing exception


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/3d21ccba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/3d21ccba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/3d21ccba

Branch: refs/heads/branch-1.1
Commit: 3d21ccba668d3c42c9bd2d276d45c55d23e0b7fe
Parents: ab4556d
Author: Manohar 
Authored: Sat Apr 15 13:42:29 2017 +0530
Committer: Manohar 
Committed: Sat Apr 15 19:38:56 2017 +0530

--
 .../core/metadata/AbsoluteTableIdentifier.java  |  5 +-
 .../spark/sql/hive/CarbonHiveMetadataUtil.scala |  5 +-
 .../deletetable/DeleteTableTestCase.scala   | 65 
 3 files changed, 71 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3d21ccba/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
 
b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
index 3791150..3c39145 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
@@ -90,8 +90,9 @@ public class AbsoluteTableIdentifier implements Serializable {
 
 String tableName = names[names.length - 1];
 String dbName = names[names.length - 2];
-String storePath = formattedTablePath.substring(0, 
formattedTablePath.lastIndexOf(dbName +
-CarbonCommonConstants.FILE_SEPARATOR + tableName));
+String storePath = formattedTablePath.substring(0,
+formattedTablePath.lastIndexOf(dbName + 
CarbonCommonConstants.FILE_SEPARATOR + tableName)
+- 1);
 
 CarbonTableIdentifier identifier =
 new CarbonTableIdentifier(dbName, tableName, 
Long.toString(System.currentTimeMillis()));

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3d21ccba/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
index 657d7de..d67ae6b 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
@@ -17,6 +17,7 @@
 package org.apache.spark.sql.hive
 
 import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.TableIdentifier
 
 import org.apache.carbondata.common.logging.LogServiceFactory
 
@@ -41,8 +42,8 @@ object CarbonHiveMetadataUtil {
   tableName: String,
   sparkSession: SparkSession): Unit = {
 try {
-  
sparkSession.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client.
-runSqlHive(s"DROP TABLE IF EXISTS $databaseName.$tableName")
+  val tabelIdentifier = TableIdentifier(tableName, Some(databaseName))
+  sparkSession.sessionState.catalog.dropTable(tabelIdentifier, true, false)
 } catch {
   case e: Exception =>
 LOGGER.audit(

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3d21ccba/integration/spark2/src/test/scala/org/apache/spark/carbondata/deletetable/DeleteTableTestCase.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/deletetable/DeleteTableTestCase.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/deletetable/DeleteTableTestCase.scala
new file mode 100644
index 000..1ab9409
--- /dev/null
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/deletetable/DeleteTableTestCase.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF 

[44/50] incubator-carbondata git commit: [CARBONDATA-931] BigDecimal and VariableLength Dimension fixes. This closes #803

[ravipesala]

[CARBONDATA-931] BigDecimal and VariableLength Dimension fixes. This closes #803


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c73e02f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c73e02f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c73e02f2

Branch: refs/heads/branch-1.1
Commit: c73e02f203403746a0cb8bbb943ca2e8e20dc1a9
Parents: f58853d 92352f3
Author: Venkata Ramana G 
Authored: Sat Apr 15 22:14:33 2017 +0530
Committer: Venkata Ramana G 
Committed: Sat Apr 15 22:14:33 2017 +0530

--
 .../store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java| 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
--

[32/50] incubator-carbondata git commit: [CARBONDATA-892] Fixed NullPointerException for 2nd level subquery. This closes #780

[ravipesala]

[CARBONDATA-892] Fixed NullPointerException for 2nd level subquery. This closes 
#780


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f27b4918
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f27b4918
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f27b4918

Branch: refs/heads/branch-1.1
Commit: f27b4918c8b9979aa69277cdc3cf73ab8b766b85
Parents: 27a1384 fc0fc7d
Author: Venkata Ramana G 
Authored: Fri Apr 14 12:58:29 2017 +0530
Committer: Venkata Ramana G 
Committed: Fri Apr 14 12:58:29 2017 +0530

--
 .../src/test/resources/temp/data1.csv   |  4 ++
 .../sql/optimizer/CarbonLateDecodeRule.scala|  6 ++-
 .../carbondata/query/SubQueryTestSuite.scala| 44 
 3 files changed, 53 insertions(+), 1 deletion(-)
--

[40/50] incubator-carbondata git commit: [CARBONDATA-927] show segment in data management doc This closes #797

[ravipesala]

[CARBONDATA-927] show segment in data management doc This closes #797


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ab4556d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ab4556d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ab4556d7

Branch: refs/heads/branch-1.1
Commit: ab4556d7102fa5c09adb14869fe75c15f5b44925
Parents: afa7887 f2f757d
Author: chenliang613 
Authored: Sat Apr 15 16:37:09 2017 +0800
Committer: chenliang613 
Committed: Sat Apr 15 16:37:09 2017 +0800

--
 docs/data-management.md | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--

[04/50] incubator-carbondata git commit: [CARBONDATA-861] Improvements in query This closes #709

[ravipesala]

[CARBONDATA-861] Improvements in query This closes #709


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4cdb7a29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4cdb7a29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4cdb7a29

Branch: refs/heads/branch-1.1
Commit: 4cdb7a290a2ac5a53fdd66fab635803f281b3f35
Parents: 00535f4 8f59a32
Author: jackylk 
Authored: Tue Apr 11 21:02:25 2017 +0800
Committer: jackylk 
Committed: Tue Apr 11 21:02:25 2017 +0800

--
 .../carbondata/core/datastore/FileHolder.java   |   4 +-
 .../impl/VariableLengthDimensionDataChunk.java  |  16 +-
 ...mpressedDimensionChunkFileBasedReaderV1.java |  14 +-
 ...mpressedDimensionChunkFileBasedReaderV2.java |  19 +--
 ...mpressedDimensionChunkFileBasedReaderV3.java |  16 +-
 ...CompressedMeasureChunkFileBasedReaderV1.java |  16 +-
 ...CompressedMeasureChunkFileBasedReaderV2.java |  19 +--
 ...CompressedMeasureChunkFileBasedReaderV3.java |  34 ++--
 .../chunk/store/DimensionDataChunkStore.java|   9 ++
 .../SafeAbsractDimensionDataChunkStore.java |   4 +
 ...feVariableLengthDimensionDataChunkStore.java |  36 -
 .../UnsafeAbstractDimensionDataChunkStore.java  |   4 +
 ...afeVariableLengthDimesionDataChunkStore.java |  12 ++
 .../core/datastore/impl/DFSFileHolderImpl.java  |   8 +-
 .../core/datastore/impl/FileHolderImpl.java |   8 +-
 .../DictionaryBasedVectorResultCollector.java   |   1 -
 .../executor/impl/AbstractQueryExecutor.java|   2 +-
 .../core/scan/executor/util/QueryUtil.java  |   2 +-
 .../vector/MeasureDataVectorProcessor.java  | 158 +--
 .../apache/carbondata/core/util/ByteUtil.java   |  29 
 .../apache/carbondata/core/util/CarbonUtil.java |  12 +-
 .../carbondata/examples/CompareTest.scala   |   4 +
 .../apache/carbondata/spark/CarbonOption.scala  |   2 +
 .../VectorizedCarbonRecordReader.java   |   2 +-
 .../spark/sql/CarbonCatalystOperators.scala |   2 +-
 .../spark/sql/CarbonDataFrameWriter.scala   |  13 +-
 .../spark/sql/CarbonDictionaryDecoder.scala |  68 
 .../execution/CarbonLateDecodeStrategy.scala|   5 +-
 28 files changed, 328 insertions(+), 191 deletions(-)
--

[24/50] incubator-carbondata git commit: [CARBONDATA-898]NullPointerException is getting thrown when rename table and select query is run concurrently. This closes #730

[ravipesala]

[CARBONDATA-898]NullPointerException is getting thrown when rename table and 
select query is run concurrently. This closes #730


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/84789391
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/84789391
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/84789391

Branch: refs/heads/branch-1.1
Commit: 8478939119462b7c071144d4bd2e0d239722992e
Parents: f7d7e41 bcd2839
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 13 16:21:29 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 16:21:29 2017 +0530

--
 .../cache/dictionary/AbstractDictionaryCache.java   | 16 ++--
 .../apache/carbondata/hadoop/util/SchemaReader.java |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)
--

[37/50] incubator-carbondata git commit: updation in file ddl-operation-on-carbondata.md

[ravipesala]

updation in file ddl-operation-on-carbondata.md


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ad738544
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ad738544
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ad738544

Branch: refs/heads/branch-1.1
Commit: ad738544b68ce2bbfce995c80ad1cfbbf705d6b3
Parents: 69ba223
Author: shivangi 
Authored: Wed Apr 12 13:11:09 2017 +0530
Committer: chenliang613 
Committed: Sat Apr 15 16:28:50 2017 +0800

--
 docs/ddl-operation-on-carbondata.md | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ad738544/docs/ddl-operation-on-carbondata.md
--
diff --git a/docs/ddl-operation-on-carbondata.md 
b/docs/ddl-operation-on-carbondata.md
index 2744691..d9d5718 100644
--- a/docs/ddl-operation-on-carbondata.md
+++ b/docs/ddl-operation-on-carbondata.md
@@ -68,8 +68,8 @@ The following DDL operations are supported in CarbonData :
 
Column groups with more than one column are stored in row format, 
instead of columnar format. By default, each column is a separate column group.
 ```
-TBLPROPERTIES ('COLUMN_GROUPS'='(column1, column3),
-(Column4,Column5,Column6)')
+TBLPROPERTIES ('COLUMN_GROUPS'='(column1, column2),
+(Column3,Column4,Column5)')
 ```
 
- **Table Block Size Configuration**
@@ -214,17 +214,17 @@ of columns is used.
 ```
  CREATE TABLE IF NOT EXISTS productSchema.productSalesTable (
 productNumber Int,
+saleQuantity Int,
 productName String,
 storeCity String,
 storeProvince String,
 productCategory String,
 productBatch String,
-saleQuantity Int,
 revenue Int)
STORED BY 'carbondata'
-   TBLPROPERTIES ('COLUMN_GROUPS'='(productName,productNumber)',
+   TBLPROPERTIES ('COLUMN_GROUPS'='(productNumber,saleQuantity)',
   'DICTIONARY_EXCLUDE'='productName',
-  'DICTIONARY_INCLUDE'='productNumber',
+  'DICTIONARY_INCLUDE'='productNumber,saleQuantity',
   'NO_INVERTED_INDEX'='productBatch',
   'BUCKETNUMBER'='4',
   'BUCKETCOLUMNS'='productName')

[07/50] incubator-carbondata git commit: fix some spelling mistakes in SegmentProperties.java

[ravipesala]

fix some spelling mistakes in SegmentProperties.java

fix some spelling mistakes in SegmentProperties.java


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/81ccec2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/81ccec2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/81ccec2a

Branch: refs/heads/branch-1.1
Commit: 81ccec2ab75c7504d7b5689c7098cf925acb54b9
Parents: 58dc3aa
Author: CGF 
Authored: Tue Apr 11 19:42:30 2017 +0800
Committer: chenliang613 
Committed: Wed Apr 12 00:42:56 2017 +0530

--
 .../core/datastore/block/SegmentProperties.java | 24 ++--
 1 file changed, 12 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/81ccec2a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 6c7d014..3bc208d 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -372,7 +372,7 @@ public class SegmentProperties {
   isComplexDimensionStarted = true;
   int previouseOrdinal = dimensonOrdinal;
   dimensonOrdinal =
-  readAllComplexTypeChildrens(dimensonOrdinal, 
columnSchema.getNumberOfChild(),
+  readAllComplexTypeChildren(dimensonOrdinal, 
columnSchema.getNumberOfChild(),
   columnsInTable, carbonDimension, complexTypeOrdinal);
   int numberOfChildrenDimensionAdded = dimensonOrdinal - 
previouseOrdinal;
   for (int i = 0; i < numberOfChildrenDimensionAdded; i++) {
@@ -418,25 +418,25 @@ public class SegmentProperties {
* @param parentDimension
* @return
*/
-  private int readAllComplexTypeChildrens(int dimensionOrdinal, int childCount,
+  private int readAllComplexTypeChildren(int dimensionOrdinal, int childCount,
   List listOfColumns, CarbonDimension parentDimension,
-  int complexDimensionOrdianl) {
+  int complexDimensionOrdinal) {
 for (int i = 0; i < childCount; i++) {
   ColumnSchema columnSchema = listOfColumns.get(dimensionOrdinal);
   if (columnSchema.isDimensionColumn()) {
 if (columnSchema.getNumberOfChild() > 0) {
   CarbonDimension complexDimension =
   new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
-  complexDimensionOrdianl++);
+  complexDimensionOrdinal++);
   
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
   parentDimension.getListOfChildDimensions().add(complexDimension);
   dimensionOrdinal =
-  readAllComplexTypeChildrens(dimensionOrdinal, 
columnSchema.getNumberOfChild(),
-  listOfColumns, complexDimension, complexDimensionOrdianl);
+  readAllComplexTypeChildren(dimensionOrdinal, 
columnSchema.getNumberOfChild(),
+  listOfColumns, complexDimension, complexDimensionOrdinal);
 } else {
   parentDimension.getListOfChildDimensions().add(
   new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
-  complexDimensionOrdianl++));
+  complexDimensionOrdinal++));
 }
   }
 }
@@ -447,18 +447,18 @@ public class SegmentProperties {
* Read all primitive/complex children and set it as list of child carbon 
dimension to parent
* dimension
*/
-  private int assignComplexOrdinal(CarbonDimension parentDimension, int 
complexDimensionOrdianl) {
+  private int assignComplexOrdinal(CarbonDimension parentDimension, int 
complexDimensionOrdinal) {
 for (int i = 0; i < parentDimension.getNumberOfChild(); i++) {
   CarbonDimension dimension = 
parentDimension.getListOfChildDimensions().get(i);
   if (dimension.getNumberOfChild() > 0) {
-dimension.setComplexTypeOridnal(++complexDimensionOrdianl);
-complexDimensionOrdianl = assignComplexOrdinal(dimension, 
complexDimensionOrdianl);
+dimension.setComplexTypeOridnal(++complexDimensionOrdinal);
+complexDimensionOrdinal = assignComplexOrdinal(dimension, 
complexDimensionOrdinal);
   } else {
 parentDimension.getListOfChildDimensions().get(i)
-.setComplexTypeOridnal(++complexDimensionOrdianl);
+.setComplexTypeOridnal(++complexDimensionOrdinal);
   }
 }
-return 

[48/50] incubator-carbondata git commit: [CARBONDATA-926] Changed max columns from static value to configurable value in Load DDL.This closes #802

[ravipesala]

[CARBONDATA-926] Changed max columns from static value to configurable value in 
Load DDL.This closes #802


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4397d059
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4397d059
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4397d059

Branch: refs/heads/branch-1.1
Commit: 4397d05999393a43676b5cc56621268c85304e48
Parents: a8ed450 0a09472
Author: Venkata Ramana G 
Authored: Mon Apr 17 13:35:49 2017 +0530
Committer: Venkata Ramana G 
Committed: Mon Apr 17 13:35:49 2017 +0530

--
 .../hadoop/test/util/StoreCreator.java  |  3 ++
 .../TestDataLoadWithColumnsMoreThanSchema.scala | 36 ++---
 .../dataload/TestLoadDataWithHiveSyntax.scala   |  2 +-
 .../carbondata/spark/rdd/CarbonMergerRDD.scala  |  3 ++
 .../carbondata/spark/util/CommonUtil.scala  | 57 +++-
 .../execution/command/carbonTableSchema.scala   |  4 +-
 .../dataload/SparkDatasourceSuite.scala |  1 -
 .../util/ExternalColumnDictionaryTestCase.scala |  1 +
 ...GlobalDictionaryUtilConcurrentTestCase.scala | 12 +++--
 .../util/GlobalDictionaryUtilTestCase.scala |  1 +
 .../execution/command/carbonTableSchema.scala   |  5 +-
 .../processing/csvload/CSVInputFormat.java  | 25 -
 .../processing/model/CarbonLoadModel.java   |  1 +
 .../newflow/CarbonDataLoadConfiguration.java| 12 +
 .../carbondata/processing/StoreCreator.java |  3 ++
 .../processing/csvload/CSVInputFormatTest.java  |  2 +
 16 files changed, 136 insertions(+), 32 deletions(-)
--

[21/50] incubator-carbondata git commit: result_size in query statistics is not giving valid row count if vector reader is enabled.

[ravipesala]

result_size in query statistics is not giving valid row count if vector reader 
is enabled.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ec2d742f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ec2d742f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ec2d742f

Branch: refs/heads/branch-1.1
Commit: ec2d742f2e479e40883b92df014a8b260d50e526
Parents: f4fc651
Author: nareshpr <prnaresh.nar...@gmail.com>
Authored: Wed Apr 12 19:53:21 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 16:14:09 2017 +0530

--
 .../carbondata/hadoop/AbstractRecordReader.java | 45 
 .../carbondata/hadoop/CarbonRecordReader.java   |  5 ++-
 .../carbondata/spark/rdd/CarbonScanRDD.scala| 13 +-
 .../VectorizedCarbonRecordReader.java   | 11 +++--
 4 files changed, 58 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ec2d742f/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java
new file mode 100644
index 000..e571ccf
--- /dev/null
+++ 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.hadoop;
+
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+
+import org.apache.hadoop.mapreduce.RecordReader;
+
+/**
+ * This class will have all the common methods for vector and row based reader
+ */
+public abstract class AbstractRecordReader extends RecordReader<Void, T> {
+
+  protected int rowCount = 0;
+
+  /**
+   * This method will log query result count and querytime
+   * @param recordCount
+   * @param recorder
+   */
+  public void logStatistics(int recordCount, QueryStatisticsRecorder recorder) 
{
+// result size
+QueryStatistic queryStatistic = new QueryStatistic();
+queryStatistic.addCountStatistic(QueryStatisticsConstants.RESULT_SIZE, 
recordCount);
+recorder.recordStatistics(queryStatistic);
+// print executor query statistics for each task_id
+recorder.logStatisticsAsTableExecutor();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ec2d742f/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
index 27c8b2f..26b269a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
@@ -33,13 +33,12 @@ import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
 
 import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 
 /**
  * Reads the data from Carbon store.
  */
-public class CarbonRecordReader extends RecordReader<Void, T> {
+public class CarbonRecordReader extends AbstractRecordReader {
 
   private QueryModel queryModel;
 
@@ -92,6 +91,7 @@ public class CarbonRecordReader extends RecordReader<Void, 
T> {
   }
 
   @Override public T getCurrentValue() throws IOException, 
InterruptedException {
+rowCount += 1;
 return readSupport.readRow(carbonIterator.next());
   }
 
@@ -101,6 +101,7 @@ public class CarbonRecordReader extends 
RecordReader<Void

[02/50] incubator-carbondata git commit: [CARBONDATA-887] lazy rdd iterator for InsertInto. This closes #765

[ravipesala]

[CARBONDATA-887] lazy rdd iterator for InsertInto. This closes #765


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/00535f4a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/00535f4a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/00535f4a

Branch: refs/heads/branch-1.1
Commit: 00535f4a9473d2879f4852d52fbcd5e07b695f52
Parents: d51387b e52e641
Author: Venkata Ramana G 
Authored: Tue Apr 11 14:11:31 2017 +0530
Committer: Venkata Ramana G 
Committed: Tue Apr 11 14:11:31 2017 +0530

--
 .../spark/rdd/NewCarbonDataLoadRDD.scala| 100 +++
 1 file changed, 81 insertions(+), 19 deletions(-)
--

[45/50] incubator-carbondata git commit: fixed variable length filter query with empty data

[ravipesala]

fixed variable length filter query with empty data


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/0729379a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/0729379a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/0729379a

Branch: refs/heads/branch-1.1
Commit: 0729379a43d72d0e2abd2e15ab3ad5d224faedc6
Parents: c73e02f
Author: vpp9380 
Authored: Sat Apr 15 17:01:48 2017 +0530
Committer: Venkata Ramana G 
Committed: Sat Apr 15 22:18:42 2017 +0530

--
 .../UnsafeVariableLengthDimesionDataChunkStore.java   | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0729379a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
index f5222fe..2b84449 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -171,7 +171,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
   /**
* to compare the two byte array
*
-   * @param indexindex of first byte array
+   * @param index index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
@@ -199,20 +199,20 @@ public class UnsafeVariableLengthDimesionDataChunkStore
 }
 // as this class handles this variable length data, so filter value can be
 // smaller or bigger than than actual data, so we need to take the smaller 
length
-int compareResult = 0;
-int compareLength = length < compareValue.length ? length : 
compareValue.length;
+int compareResult;
+int compareLength = Math.min(length , compareValue.length);
 for (int i = 0; i < compareLength; i++) {
   compareResult = 
(CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
   dataPageMemoryBlock.getBaseOffset() + currentDataOffset) & 0xff) - 
(compareValue[i]
   & 0xff);
   // if compare result is not equal we can break
   if (compareResult != 0) {
-break;
+return compareResult;
   }
   // increment the offset by one as comparison is done byte by byte
   currentDataOffset++;
 }
-return compareResult;
+return length - compareValue.length;
   }
 
 }

[28/50] incubator-carbondata git commit: [CARBONDATA-914] Clear BTree and Dictionary instances from LRU cache on table drop. This closes #788

[ravipesala]

[CARBONDATA-914] Clear BTree and Dictionary instances from LRU cache on table 
drop. This closes #788


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/51f96299
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/51f96299
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/51f96299

Branch: refs/heads/branch-1.1
Commit: 51f96299ad4b14baf689e05cb9d74d39d2089402
Parents: 144620d 673f8c2
Author: Venkata Ramana G 
Authored: Thu Apr 13 22:29:30 2017 +0530
Committer: Venkata Ramana G 
Committed: Thu Apr 13 22:29:30 2017 +0530

--
 .../core/cache/dictionary/ManageDictionary.java | 108 -
 .../dictionary/ManageDictionaryAndBTree.java| 158 +++
 .../carbondata/hadoop/CacheAccessClient.java|  23 ++-
 .../carbondata/hadoop/CarbonInputFormat.java| 153 +-
 .../spark/rdd/AlterTableDropColumnRDD.scala |   4 +-
 .../execution/command/carbonTableSchema.scala   |   8 +-
 .../carbondata/CarbonDataSourceSuite.scala  |   2 +-
 7 files changed, 269 insertions(+), 187 deletions(-)
--

[15/50] incubator-carbondata git commit: Removed redundant fields from table in configuration-parameters.md

[ravipesala]

Removed redundant fields from table in configuration-parameters.md


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d8590b16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d8590b16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d8590b16

Branch: refs/heads/branch-1.1
Commit: d8590b16a094ea1afef84c85d61fb334e8b5730e
Parents: a5eb323
Author: jatin 
Authored: Tue Apr 11 13:38:10 2017 +0530
Committer: chenliang613 
Committed: Wed Apr 12 18:03:35 2017 +0530

--
 docs/configuration-parameters.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d8590b16/docs/configuration-parameters.md
--
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index 4f454e6..e4f8f33 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -131,7 +131,7 @@ This section provides the details of all the configurations 
required for CarbonD
 | Parameter | Default Value | Description |
 
|---|-|---|
 | high.cardinality.identify.enable | true | If the parameter is true, the high 
cardinality columns of the dictionary code are automatically recognized and 
these columns will not be used as global dictionary encoding. If the parameter 
is false, all dictionary encoding columns are used as dictionary encoding. The 
high cardinality column must meet the following requirements: value of 
cardinality > configured value of high.cardinalityEqually, the value of 
cardinality is higher than the threshold.value of cardinality/ row number x 100 
> configured value of high.cardinality.row.count.percentageEqually, the ratio 
of the cardinality value to data row number is higher than the configured 
percentage. |
-| high.cardinality.threshold | 100 | high.cardinality.threshold | 100 
| It is a threshold to identify high cardinality of the columns.If the value of 
columns' cardinality > the configured value, then the columns are excluded from 
dictionary encoding. |
+| high.cardinality.threshold | 100 | It is a threshold to identify high 
cardinality of the columns.If the value of columns' cardinality > the 
configured value, then the columns are excluded from dictionary encoding. |
 | high.cardinality.row.count.percentage | 80 | Percentage to identify whether 
column cardinality is more than configured percent of total row 
count.Configuration value formula:Value of cardinality/ row number x 100 > 
configured value of high.cardinality.row.count.percentageThe value of the 
parameter must be larger than 0. |
 | carbon.cutOffTimestamp | 1970-01-01 05:30:00 | Sets the start date for 
calculating the timestamp. Java counts the number of milliseconds from start of 
"1970-01-01 00:00:00". This property is used to customize the start of 
position. For example "2000-01-01 00:00:00". The date must be in the form 
"carbon.timestamp.format". NOTE: The CarbonData supports data store up to 68 
years from the cut-off time defined. For example, if the cut-off time is 
1970-01-01 05:30:00, then the data can be stored up to 2038-01-01 05:30:00. |
 | carbon.timegranularity | SECOND | The property used to set the data 
granularity level DAY, HOUR, MINUTE, or SECOND. |

[50/50] incubator-carbondata git commit: [CARBONDATA-949] null value handled during compaction This closes #811

[ravipesala]

[CARBONDATA-949] null value handled during compaction This closes #811


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/dbf76485
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/dbf76485
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/dbf76485

Branch: refs/heads/branch-1.1
Commit: dbf76485f9964cce38118bc5a81d41050f9592d1
Parents: 4397d05 8ed7931
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Tue Apr 18 13:43:03 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Tue Apr 18 13:43:03 2017 +0530

--
 .../rowreader/AddColumnTestCases.scala  | 22 
 .../merger/CompactionResultSortProcessor.java   |  7 ---
 2 files changed, 26 insertions(+), 3 deletions(-)
--

[34/50] incubator-carbondata git commit: [CARBONDATA-863] Fixed column clean up during altertable add column failure. This closes #777

[ravipesala]

[CARBONDATA-863] Fixed column clean up during altertable add column failure. 
This closes #777


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/19b9223e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/19b9223e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/19b9223e

Branch: refs/heads/branch-1.1
Commit: 19b9223eb08b7673144ba549d0820adf243c01cc
Parents: f27b491 36112fa
Author: Venkata Ramana G 
Authored: Fri Apr 14 16:23:58 2017 +0530
Committer: Venkata Ramana G 
Committed: Fri Apr 14 16:23:58 2017 +0530

--
 .../spark/rdd/AlterTableAddColumnRDD.scala  |  6 +-
 .../execution/command/carbonTableSchema.scala   |  8 +--
 .../execution/command/AlterTableCommands.scala  | 62 
 .../org/apache/spark/util/AlterTableUtil.scala  | 16 +++--
 .../restructure/AlterTableRevertTestCase.scala  | 19 +-
 5 files changed, 68 insertions(+), 43 deletions(-)
--

[42/50] incubator-carbondata git commit: [CARBONDATA-930] Drop table IS throwing exception. This closes #801

[ravipesala]

[CARBONDATA-930] Drop table IS throwing exception. This closes #801


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f58853d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f58853d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f58853d2

Branch: refs/heads/branch-1.1
Commit: f58853d2f7fe8a6f665d64c2ffb4d65d6e3e7f39
Parents: ab4556d 3d21ccb
Author: Venkata Ramana G 
Authored: Sat Apr 15 20:06:02 2017 +0530
Committer: Venkata Ramana G 
Committed: Sat Apr 15 20:06:02 2017 +0530

--
 .../core/metadata/AbsoluteTableIdentifier.java  |  5 +-
 .../spark/sql/hive/CarbonHiveMetadataUtil.scala |  5 +-
 .../deletetable/DeleteTableTestCase.scala   | 65 
 3 files changed, 71 insertions(+), 4 deletions(-)
--

[16/50] incubator-carbondata git commit: [CARBONDATA-897] Removed redundant fields from table in configuration-parameters.md This closes #776

[ravipesala]

[CARBONDATA-897] Removed redundant fields from table in 
configuration-parameters.md This closes #776


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/11df4513
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/11df4513
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/11df4513

Branch: refs/heads/branch-1.1
Commit: 11df4513bd88d053ea0d5d41ab9919d1f47ebbfd
Parents: a5eb323 d8590b1
Author: chenliang613 
Authored: Wed Apr 12 18:04:01 2017 +0530
Committer: chenliang613 
Committed: Wed Apr 12 18:04:01 2017 +0530

--
 docs/configuration-parameters.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--

[23/50] incubator-carbondata git commit: NullPointer is getting thrown when rename table and select query is fired concurrently

[ravipesala]

NullPointer is getting thrown when rename table and select query is fired 
concurrently


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bcd28391
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bcd28391
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bcd28391

Branch: refs/heads/branch-1.1
Commit: bcd28391a6052954994513b0583397bc9c50ae8d
Parents: f7d7e41
Author: nareshpr <prnaresh.nar...@gmail.com>
Authored: Wed Apr 5 12:36:45 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 16:21:01 2017 +0530

--
 .../cache/dictionary/AbstractDictionaryCache.java   | 16 ++--
 .../apache/carbondata/hadoop/util/SchemaReader.java |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bcd28391/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
 
b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
index cc4fc14..d7c25f1 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -126,15 +126,19 @@ public abstract class AbstractDictionaryCachehttp://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bcd28391/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
index 022b3bb..f6c9e59 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
@@ -67,7 +67,7 @@ public class SchemaReader {
   return CarbonMetadata.getInstance().getCarbonTable(
   identifier.getCarbonTableIdentifier().getTableUniqueName());
 } else {
-  return null;
+  throw new IOException("File does not exist: " + schemaFilePath);
 }
   }
 }

[12/50] incubator-carbondata git commit: [CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured with the executor lru cache size. This closes #774

[ravipesala]

[CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured 
with the executor lru cache size. This closes #774


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ff182b76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ff182b76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ff182b76

Branch: refs/heads/branch-1.1
Commit: ff182b769d556743ed2c95bf7807ed703dc4d6c6
Parents: 3966f99 99582a7
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Wed Apr 12 14:11:10 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 12 14:11:10 2017 +0530

--
 .../core/cache/CacheProviderTest.java   | 53 
 .../scala/org/apache/spark/sql/CarbonEnv.scala  |  1 +
 2 files changed, 54 insertions(+)
--

[09/50] incubator-carbondata git commit: CARBONDATA-880 Path is getting printed in the explain extended DDL. this should not get printed. so overriding the toString method.

[ravipesala]

CARBONDATA-880
Path is getting printed in the explain extended DDL. this should not get 
printed. so overriding the toString method.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/591f9f69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/591f9f69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/591f9f69

Branch: refs/heads/branch-1.1
Commit: 591f9f692cd2e7561e315dae60bf0c482ae46a01
Parents: aea9ed6
Author: ravikiran <ravikiran.sn...@gmail.com>
Authored: Thu Apr 6 18:30:39 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 12 13:55:01 2017 +0530

--
 .../org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala   | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/591f9f69/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index b9e8682..322e1ae 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -74,4 +74,9 @@ case class CarbonDatasourceHadoopRelation(
   }
   override def unhandledFilters(filters: Array[Filter]): Array[Filter] = new 
Array[Filter](0)
 
+  override def toString: String = {
+"CarbonDatasourceHadoopRelation [ " + "Database name :" + 
carbonTable.getDatabaseName +
+", " + "Table name :" + carbonTable.getFactTableName + ", Schema :" + 
tableSchema + " ]"
+  }
+
 }

[06/50] incubator-carbondata git commit: [CARBONDATA-660] Bad Records Logs and Raw CSVs should get display under segment id instead of Tasks id This closes #775

[ravipesala]

[CARBONDATA-660] Bad Records Logs and Raw CSVs should get display under segment 
id instead of Tasks id  This closes #775


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/58dc3aa6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/58dc3aa6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/58dc3aa6

Branch: refs/heads/branch-1.1
Commit: 58dc3aa646c2ba470828a7cdd3f101f2f63239e4
Parents: 4cdb7a2 8d733f5
Author: jackylk 
Authored: Tue Apr 11 21:18:06 2017 +0800
Committer: jackylk 
Committed: Tue Apr 11 21:18:06 2017 +0800

--
 .../steps/DataConverterProcessorWithBucketingStepImpl.java| 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--

[13/50] incubator-carbondata git commit: Update dml-operation-on-carbondata.md

[ravipesala]

Update dml-operation-on-carbondata.md

Update dml-operation-on-carbondata.md


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b5c9985e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b5c9985e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b5c9985e

Branch: refs/heads/branch-1.1
Commit: b5c9985ec3e210b64c28eff1b7070be7f387fd7a
Parents: ff182b7
Author: chenerlu 
Authored: Wed Apr 12 10:03:29 2017 +0800
Committer: chenliang613 
Committed: Wed Apr 12 17:36:08 2017 +0530

--
 docs/dml-operation-on-carbondata.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5c9985e/docs/dml-operation-on-carbondata.md
--
diff --git a/docs/dml-operation-on-carbondata.md 
b/docs/dml-operation-on-carbondata.md
index 6f67543..579b9cb 100644
--- a/docs/dml-operation-on-carbondata.md
+++ b/docs/dml-operation-on-carbondata.md
@@ -290,8 +290,8 @@ This command will allow to delete the CarbonData segment(s) 
from the store based
 The segment created before the particular date will be removed from the 
specific stores.
 
 ```
-DELETE FROM TABLE [schema_name.]table_name 
-WHERE[DATE_FIELD]BEFORE [DATE_VALUE]
+DELETE SEGMENTS FROM TABLE [db_name.]table_name 
+WHERE STARTTIME BEFORE DATE_VALUE
 ```
 
 ### Parameter Description

[1/2] incubator-carbondata git commit: null value handled during compaction

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 4397d0599 -> dbf76485f


null value handled during compaction


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8ed7931b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8ed7931b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8ed7931b

Branch: refs/heads/master
Commit: 8ed7931bae6858a47c4989dde7223b20eecc7803
Parents: 4397d05
Author: rahulforallp 
Authored: Tue Apr 18 12:26:51 2017 +0530
Committer: rahulforallp 
Committed: Tue Apr 18 12:26:51 2017 +0530

--
 .../rowreader/AddColumnTestCases.scala  | 22 
 .../merger/CompactionResultSortProcessor.java   |  7 ---
 2 files changed, 26 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8ed7931b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
index e6fb265..06f480b 100644
--- 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
@@ -131,6 +131,28 @@ class AddColumnTestCases extends QueryTest with 
BeforeAndAfterAll {
 sql("DROP TABLE IF EXISTS carbon_table")
   }
 
+
+  test("test add column compaction") {
+sql("DROP TABLE IF EXISTS carbon_table")
+sql(
+  "CREATE TABLE carbon_table(intField int,stringField string,charField 
string,timestampField " +
+  "timestamp)STORED BY 'carbondata' TBLPROPERTIES" +
+  "('DICTIONARY_EXCLUDE'='charField')")
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO 
TABLE carbon_table " +
+
s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO 
TABLE carbon_table " +
+
s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO 
TABLE carbon_table " +
+
s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO 
TABLE carbon_table " +
+
s"options('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+sql("Alter table carbon_table add columns(decimalField decimal(6,2))")
+
+sql("Alter table carbon_table compact 'minor'")
+
+sql("DROP TABLE IF EXISTS carbon_table")
+  }
+
   override def afterAll {
 sql("DROP TABLE IF EXISTS addcolumntest")
 sql("drop table if exists hivetable")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8ed7931b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
 
b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index ebf3683..8c1f577 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -184,8 +184,7 @@ public class CompactionResultSortProcessor extends 
AbstractResultProcessor {
*
* @param resultIteratorList
*/
-  private void processResult(List resultIteratorList)
-  throws Exception {
+  private void processResult(List resultIteratorList) 
throws Exception {
 for (RawResultIterator resultIterator : resultIteratorList) {
   while (resultIterator.hasNext()) {
 addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));
@@ -250,7 +249,9 @@ public class CompactionResultSortProcessor extends 
AbstractResultProcessor {
   private Object getConvertedMeasureValue(Object value, char aggType) {
 switch (aggType) {
   case CarbonCommonConstants.BIG_DECIMAL_MEASURE:
-value = ((org.apache.spark.sql.types.Decimal) 
value).toJavaBigDecimal();
+if (value != null) {
+  value = ((org.apache.spark.sql.types.Decimal) 
value).toJavaBigDecimal();

[2/2] incubator-carbondata git commit: [CARBONDATA-898]NullPointerException is getting thrown when rename table and select query is run concurrently. This closes #730

[ravipesala]

[CARBONDATA-898]NullPointerException is getting thrown when rename table and 
select query is run concurrently. This closes #730


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/84789391
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/84789391
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/84789391

Branch: refs/heads/master
Commit: 8478939119462b7c071144d4bd2e0d239722992e
Parents: f7d7e41 bcd2839
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 13 16:21:29 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 16:21:29 2017 +0530

--
 .../cache/dictionary/AbstractDictionaryCache.java   | 16 ++--
 .../apache/carbondata/hadoop/util/SchemaReader.java |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)
--

[1/2] incubator-carbondata git commit: NullPointer is getting thrown when rename table and select query is fired concurrently

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master f7d7e41ef -> 847893911


NullPointer is getting thrown when rename table and select query is fired 
concurrently


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bcd28391
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bcd28391
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bcd28391

Branch: refs/heads/master
Commit: bcd28391a6052954994513b0583397bc9c50ae8d
Parents: f7d7e41
Author: nareshpr <prnaresh.nar...@gmail.com>
Authored: Wed Apr 5 12:36:45 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 16:21:01 2017 +0530

--
 .../cache/dictionary/AbstractDictionaryCache.java   | 16 ++--
 .../apache/carbondata/hadoop/util/SchemaReader.java |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bcd28391/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
 
b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
index cc4fc14..d7c25f1 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -126,15 +126,19 @@ public abstract class AbstractDictionaryCachehttp://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bcd28391/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
index 022b3bb..f6c9e59 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
@@ -67,7 +67,7 @@ public class SchemaReader {
   return CarbonMetadata.getInstance().getCarbonTable(
   identifier.getCarbonTableIdentifier().getTableUniqueName());
 } else {
-  return null;
+  throw new IOException("File does not exist: " + schemaFilePath);
 }
   }
 }

[1/2] incubator-carbondata git commit: result_size in query statistics is not giving valid row count if vector reader is enabled.

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master f4fc65199 -> f7d7e41ef


result_size in query statistics is not giving valid row count if vector reader 
is enabled.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ec2d742f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ec2d742f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ec2d742f

Branch: refs/heads/master
Commit: ec2d742f2e479e40883b92df014a8b260d50e526
Parents: f4fc651
Author: nareshpr <prnaresh.nar...@gmail.com>
Authored: Wed Apr 12 19:53:21 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 16:14:09 2017 +0530

--
 .../carbondata/hadoop/AbstractRecordReader.java | 45 
 .../carbondata/hadoop/CarbonRecordReader.java   |  5 ++-
 .../carbondata/spark/rdd/CarbonScanRDD.scala| 13 +-
 .../VectorizedCarbonRecordReader.java   | 11 +++--
 4 files changed, 58 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ec2d742f/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java
new file mode 100644
index 000..e571ccf
--- /dev/null
+++ 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.hadoop;
+
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+
+import org.apache.hadoop.mapreduce.RecordReader;
+
+/**
+ * This class will have all the common methods for vector and row based reader
+ */
+public abstract class AbstractRecordReader extends RecordReader<Void, T> {
+
+  protected int rowCount = 0;
+
+  /**
+   * This method will log query result count and querytime
+   * @param recordCount
+   * @param recorder
+   */
+  public void logStatistics(int recordCount, QueryStatisticsRecorder recorder) 
{
+// result size
+QueryStatistic queryStatistic = new QueryStatistic();
+queryStatistic.addCountStatistic(QueryStatisticsConstants.RESULT_SIZE, 
recordCount);
+recorder.recordStatistics(queryStatistic);
+// print executor query statistics for each task_id
+recorder.logStatisticsAsTableExecutor();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ec2d742f/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
index 27c8b2f..26b269a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
@@ -33,13 +33,12 @@ import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
 
 import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 
 /**
  * Reads the data from Carbon store.
  */
-public class CarbonRecordReader extends RecordReader<Void, T> {
+public class CarbonRecordReader extends AbstractRecordReader {
 
   private QueryModel queryModel;
 
@@ -92,6 +91,7 @@ public class CarbonRecordReader extends RecordReader<Void, 
T> {
   }
 
   @Override public T getCurrentValue() throws IOException, 
InterruptedException {
+rowCount += 1;
 return readSupport.readRow(carbonIter

[2/2] incubator-carbondata git commit: [CARBONDATA-919]result_size in query statistics is not giving valid row count if vector reader is enabled. This closes #790

[ravipesala]

[CARBONDATA-919]result_size in query statistics is not giving valid row count 
if vector reader is enabled. This closes #790


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f7d7e41e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f7d7e41e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f7d7e41e

Branch: refs/heads/master
Commit: f7d7e41efd987d341b5a40085ec550d3ee265e23
Parents: f4fc651 ec2d742
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 13 16:15:09 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 16:15:09 2017 +0530

--
 .../carbondata/hadoop/AbstractRecordReader.java | 45 
 .../carbondata/hadoop/CarbonRecordReader.java   |  5 ++-
 .../carbondata/spark/rdd/CarbonScanRDD.scala| 13 +-
 .../VectorizedCarbonRecordReader.java   | 11 +++--
 4 files changed, 58 insertions(+), 16 deletions(-)
--

[2/2] incubator-carbondata git commit: [CARBONDATA-916] Major Compaction : Updating proper schema update time stamp. This closes #789

[ravipesala]

[CARBONDATA-916] Major Compaction : Updating proper schema update time stamp. 
This closes #789


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9efcacda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9efcacda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9efcacda

Branch: refs/heads/master
Commit: 9efcacdac2ff9c07107e7a6b6fcf299c0bc817a6
Parents: 11df451 ebc5ee9
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 13 15:37:28 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 15:37:28 2017 +0530

--
 .../MajorCompactionIgnoreInMinorTest.scala  | 53 +---
 .../spark/rdd/CarbonDataRDDFactory.scala| 11 ++--
 .../store/CarbonFactDataHandlerModel.java   |  1 +
 3 files changed, 54 insertions(+), 11 deletions(-)
--

[1/2] incubator-carbondata git commit: Updating proper schema update time stamp

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 11df4513b -> 9efcacdac


Updating proper schema update time stamp


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ebc5ee97
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ebc5ee97
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ebc5ee97

Branch: refs/heads/master
Commit: ebc5ee973cf6e55194d9804a7a70b2011505339a
Parents: 11df451
Author: Manohar <manohar.craz...@gmail.com>
Authored: Wed Apr 12 16:22:26 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 13 15:36:19 2017 +0530

--
 .../MajorCompactionIgnoreInMinorTest.scala  | 53 +---
 .../spark/rdd/CarbonDataRDDFactory.scala| 11 ++--
 .../store/CarbonFactDataHandlerModel.java   |  1 +
 3 files changed, 54 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ebc5ee97/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index a879598..9d2cf96 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -35,6 +35,10 @@ import org.apache.carbondata.hadoop.CacheClient
   */
 class MajorCompactionIgnoreInMinorTest extends QueryTest with 
BeforeAndAfterAll {
 
+  val csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
+  val csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
+  val csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
+
   override def beforeAll {
 
CarbonProperties.getInstance().addProperty("carbon.compaction.level.threshold", 
"2,2")
 sql("drop table if exists  ignoremajor")
@@ -46,12 +50,6 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest 
with BeforeAndAfterAll
 "phonetype String, serialname String, salary Int) STORED BY 
'org.apache.carbondata" +
 ".format'"
 )
-
-
-val csvFilePath1 = s"$resourcesPath/compaction/compaction1.csv"
-val csvFilePath2 = s"$resourcesPath/compaction/compaction2.csv"
-val csvFilePath3 = s"$resourcesPath/compaction/compaction3.csv"
-
 sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE ignoremajor 
OPTIONS" +
   "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
 )
@@ -149,8 +147,51 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest 
with BeforeAndAfterAll
 
   }
 
+  /**
+   * Execute two major compactions sequentially
+   */
+  test("Execute two major compactions sequentially") {
+sql("drop table if exists testmajor")
+sql(
+  "CREATE TABLE IF NOT EXISTS testmajor (country String, ID Int, date 
Timestamp, name " +
+  "String, " +
+  "phonetype String, serialname String, salary Int) STORED BY 
'org.apache.carbondata" +
+  ".format'"
+)
+sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE testmajor 
OPTIONS" +
+"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+)
+sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE testmajor  
OPTIONS" +
+"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+)
+// compaction will happen here.
+sql("alter table testmajor compact 'major'")
+sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE testmajor 
OPTIONS" +
+"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+)
+sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE testmajor  
OPTIONS" +
+"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+)
+sql("alter table testmajor compact 'major'")
+val identifier = new AbsoluteTableIdentifier(
+  
CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
+  new CarbonTableIdentifier(
+CarbonCommonConstants.DATABASE_DEFAULT_NAME, "testmajor", "ttt

[2/2] incubator-carbondata git commit: [CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured with the executor lru cache size. This closes #774

[ravipesala]

[CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured 
with the executor lru cache size. This closes #774


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ff182b76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ff182b76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ff182b76

Branch: refs/heads/master
Commit: ff182b769d556743ed2c95bf7807ed703dc4d6c6
Parents: 3966f99 99582a7
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Wed Apr 12 14:11:10 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 12 14:11:10 2017 +0530

--
 .../core/cache/CacheProviderTest.java   | 53 
 .../scala/org/apache/spark/sql/CarbonEnv.scala  |  1 +
 2 files changed, 54 insertions(+)
--

[1/2] incubator-carbondata git commit: [CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured with the executor lru cache size.

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 3966f9927 -> ff182b769


[CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured 
with the executor lru cache size.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/99582a76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/99582a76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/99582a76

Branch: refs/heads/master
Commit: 99582a762871c8a2c6cc9f5db43718d21e13de0c
Parents: 3966f99
Author: mohammadshahidkhan <mohdshahidkhan1...@gmail.com>
Authored: Tue Apr 11 09:59:45 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 12 14:10:34 2017 +0530

--
 .../core/cache/CacheProviderTest.java   | 53 
 .../scala/org/apache/spark/sql/CarbonEnv.scala  |  1 +
 2 files changed, 54 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/99582a76/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java 
b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
index 1be5cb2..4c12259 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
@@ -17,16 +17,26 @@
 
 package org.apache.carbondata.core.cache;
 
+import java.io.IOException;
+import java.lang.reflect.Field;
+
 import org.apache.carbondata.core.cache.dictionary.Dictionary;
 import 
org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
 import org.apache.carbondata.core.cache.dictionary.ForwardDictionaryCache;
 import org.apache.carbondata.core.cache.dictionary.ReverseDictionaryCache;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.BlockIndexStore;
+import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
+import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
+import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
 import org.apache.carbondata.core.util.CarbonProperties;
 
+import org.apache.avro.Schema;
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -39,6 +49,9 @@ public class CacheProviderTest {
 // enable lru cache by setting cache size
 CarbonProperties.getInstance()
 .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, 
"10");
+// enable lru cache by setting cache size
+CarbonProperties.getInstance()
+.addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, 
"20");
   }
 
   @Test public void getInstance() throws Exception {
@@ -62,4 +75,44 @@ public class CacheProviderTest {
 assertTrue(reverseDictionaryCache instanceof ReverseDictionaryCache);
 assertFalse(reverseDictionaryCache instanceof ForwardDictionaryCache);
   }
+
+  /**
+   * to test the driver and executor lru memory configuration
+   *
+   * @throws IOException
+   * @throws NoSuchFieldException
+   * @throws IllegalAccessException
+   */
+  @Test public void driverExecutorCacheConfTest()
+  throws IOException, NoSuchFieldException, IllegalAccessException {
+// get cache provider instance
+CacheProvider cacheProvider = CacheProvider.getInstance();
+
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE,
 "true");
+Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexStore> driverCache =
+cacheProvider.createCache(CacheType.DRIVER_BTREE, "carbonStore");
+Field carbonLRUCacheField = 
SegmentTaskIndexStore.class.getDeclaredField("lruCache");
+carbonLRUCacheField.setAccessible(true);
+CarbonLRUCache carbonLRUCache = (CarbonLRUCache) 
carbonLRUCacheField.get(driverCache);
+Field lruCacheMemorySizeField = 
CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize");
+lruCacheMemorySizeField.setAccessible(true);
+long lruCacheMemorySize = (long) 
lruCacheMemorySizeField.get(carbonLRUCache);
+String driverCacheSize = CarbonProperties.getInstance()
+.getProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE);
+assertEquals(1024 * 1024 * Integer.parseInt(driverCacheSize), 
lruCacheMemorySize);
+// drop cache
+cacheProvider.dropAllCache();
+ 

[1/2] incubator-carbondata git commit: [CARBONDATA-891] Fix compilation issue of LocalFileLockTest generate new folder carbon.store

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 490901d9d -> 065381995


[CARBONDATA-891] Fix compilation issue of LocalFileLockTest generate new folder 
carbon.store


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/fce12a8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/fce12a8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/fce12a8f

Branch: refs/heads/master
Commit: fce12a8fb09209b7a4a2938dcf99835f0cef2ea1
Parents: 490901d
Author: chenliang613 
Authored: Mon Apr 10 17:21:47 2017 +0530
Committer: chenliang613 
Committed: Mon Apr 10 17:21:47 2017 +0530

--
 .../restructure/AlterTableRevertTestCase.scala | 17 +
 .../restructure/AlterTableValidationTestCase.scala | 17 +
 .../restructure/rowreader/AddColumnTestCases.scala | 17 +
 .../rowreader/ChangeDataTypeTestCases.scala| 17 +
 .../rowreader/DropColumnTestCases.scala| 17 +
 .../carbondata/lcm/locks/LocalFileLockTest.java|  8 
 6 files changed, 93 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fce12a8f/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala
index 958b426..05b79a8 100644
--- 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.spark.carbondata.restructure
 
 import java.io.File

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fce12a8f/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
index 91dd8b3..c826b0c 100644
--- 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.spark.carbondata.restructure
 
 import java.io.File

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fce12a8f/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala
--
diff --git 

[2/2] incubator-carbondata git commit: [CARBONDATA-891] Fix compilation issue of LocalFileLockTest generate new folder carbon.store. This closes #772

[ravipesala]

[CARBONDATA-891] Fix compilation issue of LocalFileLockTest generate new folder 
carbon.store. This closes #772


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/06538199
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/06538199
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/06538199

Branch: refs/heads/master
Commit: 06538199592dd49f4e1e4acc49236670838de2ca
Parents: 490901d fce12a8
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Mon Apr 10 18:33:33 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Mon Apr 10 18:33:33 2017 +0530

--
 .../restructure/AlterTableRevertTestCase.scala | 17 +
 .../restructure/AlterTableValidationTestCase.scala | 17 +
 .../restructure/rowreader/AddColumnTestCases.scala | 17 +
 .../rowreader/ChangeDataTypeTestCases.scala| 17 +
 .../rowreader/DropColumnTestCases.scala| 17 +
 .../carbondata/lcm/locks/LocalFileLockTest.java|  8 
 6 files changed, 93 insertions(+)
--

[1/4] incubator-carbondata git commit: fixed ArrayIndexOutOfBoundException for ColumnarVectorWrapper

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 6f12fbb31 -> 24d9fd2e3


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7e2e0f3d/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
--
diff --git 
a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
 
b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
index 2a9c701..f94c0b2 100644
--- 
a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
+++ 
b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
@@ -60,8 +60,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
 
   @Override public void putDecimals(int rowId, int count, Decimal value, int 
precision) {
 for (int i = 0; i < count; i++) {
-  rowId += i;
-  putDecimal(rowId, value, precision);
+  putDecimal(rowId++, value, precision);
 }
   }
 
@@ -79,8 +78,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
 
   @Override public void putBytes(int rowId, int count, byte[] value) {
 for (int i = 0; i < count; i++) {
-  rowId += i;
-  putBytes(rowId, value);
+  putBytes(rowId++, value);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7e2e0f3d/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
index 36c9d95..747af05 100644
--- 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
@@ -167,6 +167,27 @@ class AddColumnTestCases extends QueryTest with 
BeforeAndAfterAll {
 sql("drop table carbon_new")
   }
 
+  test("test to check if vector result collector is able to fetch large amount 
of data") {
+sql("DROP TABLE IF EXISTS carbon_new")
+sql(
+  """CREATE TABLE carbon_new (CUST_ID int,CUST_NAME 
String,ACTIVE_EMUI_VERSION string, DOB
+|timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 
bigint,DECIMAL_COLUMN1
+|decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, 
Double_COLUMN2
+|double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' 
TBLPROPERTIES
+|("TABLE_BLOCKSIZE"= "256 MB")""".stripMargin)
+sql("alter table carbon_new drop columns(CUST_NAME)")
+sql(s"LOAD DATA INPATH '$resourcesPath/restructure/data_2000.csv' into 
table " +
+"carbon_new OPTIONS('DELIMITER'=',' , 
'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE'," +
+
"'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+
"BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2," 
+
+"INTEGER_COLUMN1')")
+sql(
+  """alter table carbon_new add columns(CUST_NAME string) TBLPROPERTIES
+('DICTIONARY_EXCLUDE'='CUST_NAME', 
'DEFAULT.VALUE.CUST_NAME'='testuser')""")
+checkAnswer(sql("select distinct(CUST_NAME) from 
carbon_new"),Row("testuser"))
+  }
+
+
   override def afterAll {
 sql("DROP TABLE IF EXISTS addcolumntest")
 sql("drop table if exists hivetable")

[3/4] incubator-carbondata git commit: fixed ArrayIndexOutOfBoundException for ColumnarVectorWrapper

[ravipesala]

fixed ArrayIndexOutOfBoundException for ColumnarVectorWrapper


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7e2e0f3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7e2e0f3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7e2e0f3d

Branch: refs/heads/master
Commit: 7e2e0f3dea340a82611c07fc4acd2d79d7f2c8a7
Parents: 6f12fbb
Author: kunal642 
Authored: Thu Apr 6 18:35:57 2017 +0530
Committer: kunal642 
Committed: Thu Apr 6 18:35:57 2017 +0530

--
 .../test/resources/restructure/data_2000.csv| 2013 ++
 .../vectorreader/ColumnarVectorWrapper.java |6 +-
 .../vectorreader/AddColumnTestCases.scala   |   21 +
 3 files changed, 2036 insertions(+), 4 deletions(-)
--

[4/4] incubator-carbondata git commit: [CARBONDATA-877] Fixed ArrayIndexOutOfBoundException for ColumnarVectorWrapper. This closes #753

[ravipesala]

[CARBONDATA-877] Fixed ArrayIndexOutOfBoundException for ColumnarVectorWrapper. 
This closes #753


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/24d9fd2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/24d9fd2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/24d9fd2e

Branch: refs/heads/master
Commit: 24d9fd2e35526c68659c94cdaa1bcca1b2e8993b
Parents: 6f12fbb 7e2e0f3
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 20:09:45 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 20:09:45 2017 +0530

--
 .../test/resources/restructure/data_2000.csv| 2013 ++
 .../vectorreader/ColumnarVectorWrapper.java |6 +-
 .../vectorreader/AddColumnTestCases.scala   |   21 +
 3 files changed, 2036 insertions(+), 4 deletions(-)
--

[2/2] incubator-carbondata git commit: [CARBONDATA-846] Added support to revert changes if query fails. This closes #727

[ravipesala]

[CARBONDATA-846] Added support to revert changes if query fails. This closes 
#727


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/6f12fbb3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/6f12fbb3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/6f12fbb3

Branch: refs/heads/master
Commit: 6f12fbb313c330cf5a5c4c3da0a0598f8b3e7ebf
Parents: bbade2a 2a4f09b
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 17:59:13 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 17:59:13 2017 +0530

--
 .../ThriftWrapperSchemaConverterImpl.java   |   4 +-
 .../carbondata/core/util/DataTypeUtil.java  |  16 +-
 .../execution/command/carbonTableSchema.scala   |   2 +-
 .../execution/command/AlterTableCommands.scala  |  48 +++---
 .../apache/spark/sql/hive/CarbonMetastore.scala |  30 +++-
 .../org/apache/spark/util/AlterTableUtil.scala  | 163 ++-
 .../restructure/AlterTableRevertTestCase.scala  |  69 
 .../spark/sql/common/util/QueryTest.scala   |   3 +
 8 files changed, 303 insertions(+), 32 deletions(-)
--

[2/2] incubator-carbondata git commit: [CARBONDATA-847] Fixed NullPointerException for DictionaryBasedVectorResultCollector during alter table. This closes #724

[ravipesala]

[CARBONDATA-847] Fixed NullPointerException for 
DictionaryBasedVectorResultCollector during alter table. This closes #724


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/1624c5f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/1624c5f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/1624c5f7

Branch: refs/heads/master
Commit: 1624c5f7a8064eb7c149f8b38710ae7bdd90a69e
Parents: 413cb93 22be345
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 16:45:11 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 16:45:11 2017 +0530

--
 .../RestructureBasedRawResultCollector.java | 13 +++--
 .../RestructureBasedVectorResultCollector.java  |  7 ++-
 .../src/test/resources/restructure/data1.csv|  2 +-
 .../vectorreader/AddColumnTestCases.scala   | 61 
 4 files changed, 75 insertions(+), 8 deletions(-)
--

[4/7] incubator-carbondata git commit: Compaction lock should also be acquired during alter operation as alter and compaction on same table should not be allowed concurrently.

[ravipesala]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cc59b247/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
--
diff --git 
a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
 
b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
deleted file mode 100644
index 14bed0a..000
--- 
a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ /dev/null
@@ -1,1371 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.merger;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.locks.CarbonLockFactory;
-import org.apache.carbondata.core.locks.ICarbonLock;
-import org.apache.carbondata.core.locks.LockUsage;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
-import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails;
-import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
-import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
-import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
-import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl;
-import org.apache.carbondata.processing.model.CarbonLoadModel;
-import org.apache.carbondata.spark.load.CarbonLoaderUtil;
-
-/**
- * utility class for load merging.
- */
-public final class CarbonDataMergerUtil {
-  private static final LogService LOGGER =
-  LogServiceFactory.getLogService(CarbonDataMergerUtil.class.getName());
-
-  private CarbonDataMergerUtil() {
-
-  }
-
-  /**
-   * Returns the size of all the carbondata files present in the segment.
-   * @param carbonFile
-   * @return
-   */
-  private static long getSizeOfFactFileInLoad(CarbonFile carbonFile) {
-long factSize = 0;
-
-// carbon data file case.
-CarbonFile[] factFile = carbonFile.listFiles(new CarbonFileFilter() {
-
-  @Override public boolean accept(CarbonFile file) {
-return CarbonTablePath.isCarbonDataFile(file.getName());
-  }
-});
-
-for (CarbonFile fact : factFile) {
-  factSize += fact.getSize();
-}
-
-return factSize;
-  }
-
-  /**
-   * To check whether the merge property is enabled or not.
-   *
-   * @return
-   */
-
-  public static boolean checkIfAutoLoadMergingRequired() {
-// load merge is not supported as per new store format
-// moving the load merge check in early to avoid unnecessary load listing 
causing IOException
-// check whether carbons segment merging operation is enabled or not.
-// default will be false.
-String isLoadMergeEnabled = CarbonProperties.getInstance()
-.getProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
-CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE);
-if 

[2/7] incubator-carbondata git commit: Compaction lock should also be acquired during alter operation as alter and compaction on same table should not be allowed concurrently.

[ravipesala]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cc59b247/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
 
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
new file mode 100644
index 000..ee667c2
--- /dev/null
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -0,0 +1,1385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.merger;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails;
+import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
+import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.path.CarbonStorePath;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl;
+import org.apache.carbondata.processing.model.CarbonLoadModel;
+
+/**
+ * utility class for load merging.
+ */
+public final class CarbonDataMergerUtil {
+  private static final LogService LOGGER =
+  LogServiceFactory.getLogService(CarbonDataMergerUtil.class.getName());
+
+  private CarbonDataMergerUtil() {
+
+  }
+
+  /**
+   * Returns the size of all the carbondata files present in the segment.
+   * @param carbonFile
+   * @return
+   */
+  private static long getSizeOfFactFileInLoad(CarbonFile carbonFile) {
+long factSize = 0;
+
+// carbon data file case.
+CarbonFile[] factFile = carbonFile.listFiles(new CarbonFileFilter() {
+
+  @Override public boolean accept(CarbonFile file) {
+return CarbonTablePath.isCarbonDataFile(file.getName());
+  }
+});
+
+for (CarbonFile fact : factFile) {
+  factSize += fact.getSize();
+}
+
+return factSize;
+  }
+
+  /**
+   * To check whether the merge property is enabled or not.
+   *
+   * @return
+   */
+
+  public static boolean checkIfAutoLoadMergingRequired() {
+// load merge is not supported as per new store format
+// moving the load merge check in early to avoid unnecessary load listing 
causing IOException
+// check whether carbons segment merging operation is enabled or not.
+// default will be false.
+String isLoadMergeEnabled = CarbonProperties.getInstance()
+.getProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
+CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE);
+if (isLoadMergeEnabled.equalsIgnoreCase("false")) {
+  return false;
+}
+return true;
+  }
+
+  

[3/7] incubator-carbondata git commit: Compaction lock should also be acquired during alter operation as alter and compaction on same table should not be allowed concurrently.

[ravipesala]

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cc59b247/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
--
diff --git 
a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
 
b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
deleted file mode 100644
index 29aa7e7..000
--- 
a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
+++ /dev/null
@@ -1,367 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.merger;
-
-import java.io.File;
-import java.util.AbstractQueue;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.PriorityQueue;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
-import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
-import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.processing.datatypes.GenericDataType;
-import org.apache.carbondata.processing.model.CarbonLoadModel;
-import org.apache.carbondata.processing.store.CarbonDataFileAttributes;
-import org.apache.carbondata.processing.store.CarbonFactDataHandlerColumnar;
-import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
-import org.apache.carbondata.processing.store.CarbonFactHandler;
-import 
org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
-import org.apache.carbondata.spark.merger.exeception.SliceMergerException;
-
-/**
- * This is the Merger class responsible for the merging of the segments.
- */
-public class RowResultMerger {
-
-  private final String databaseName;
-  private final String tableName;
-  private final String tempStoreLocation;
-  private final String factStoreLocation;
-  private CarbonFactHandler dataHandler;
-  private List rawResultIteratorList =
-  new 
ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-  private SegmentProperties segprop;
-  /**
-   * record holder heap
-   */
-  private AbstractQueue recordHolderHeap;
-
-  private TupleConversionAdapter tupleConvertor;
-
-  private static final LogService LOGGER =
-  LogServiceFactory.getLogService(RowResultMerger.class.getName());
-
-  public RowResultMerger(List iteratorList, String 
databaseName,
-  String tableName, SegmentProperties segProp, String tempStoreLocation,
-  CarbonLoadModel loadModel, CompactionType compactionType) {
-
-CarbonDataFileAttributes carbonDataFileAttributes;
-
-this.rawResultIteratorList = iteratorList;
-// create the List of RawResultIterator.
-
-recordHolderHeap = new 
PriorityQueue(rawResultIteratorList.size(),
-new RowResultMerger.CarbonMdkeyComparator());
-
-this.segprop = segProp;
-this.tempStoreLocation = tempStoreLocation;
-
-this.factStoreLocation = loadModel.getStorePath();
-
-if (!new File(tempStoreLocation).mkdirs()) {
-  

[7/7] incubator-carbondata git commit: [CARBONDATA-780] Alter table support for compaction through sort step. This closes #703

[ravipesala]

[CARBONDATA-780] Alter table support for compaction through sort step. This 
closes #703


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/68cbe150
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/68cbe150
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/68cbe150

Branch: refs/heads/master
Commit: 68cbe1508a4703eba3de1731e77b217e604c677d
Parents: 58cd933 cc59b24
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 16:18:36 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 16:18:36 2017 +0530

--
 .../core/locks/CarbonLockFactory.java   |8 +
 .../DictionaryBasedVectorResultCollector.java   |   12 +-
 .../RestructureBasedRawResultCollector.java |2 -
 .../spark/compaction/CompactionCallable.java|   44 +
 .../carbondata/spark/load/CarbonLoaderUtil.java |   19 +-
 .../spark/merger/CarbonCompactionExecutor.java  |  211 ---
 .../spark/merger/CarbonCompactionUtil.java  |  354 -
 .../spark/merger/CarbonDataMergerUtil.java  | 1371 -
 .../merger/CarbonDataMergerUtilResult.java  |   33 -
 .../spark/merger/CompactionCallable.java|   44 -
 .../carbondata/spark/merger/CompactionType.java |   31 -
 .../spark/merger/NodeBlockRelation.java |   58 -
 .../spark/merger/NodeMultiBlockRelation.java|   57 -
 .../spark/merger/RowResultMerger.java   |  367 -
 .../carbondata/spark/merger/TableMeta.java  |   40 -
 .../spark/merger/TupleConversionAdapter.java|   67 -
 .../merger/exeception/SliceMergerException.java |   78 -
 .../spark/rdd/CarbonIUDMergerRDD.scala  |2 +-
 .../carbondata/spark/rdd/CarbonMergerRDD.scala  |   37 +-
 .../apache/carbondata/spark/rdd/Compactor.scala |3 +-
 .../spark/rdd/DataManagementFunc.scala  |3 +-
 .../execution/command/carbonTableSchema.scala   |2 +-
 .../spark/rdd/CarbonDataRDDFactory.scala|2 +-
 .../sql/CarbonDatasourceHadoopRelation.scala|2 +-
 .../spark/sql/CarbonDatasourceRelation.scala|2 +-
 .../sql/execution/command/IUDCommands.scala |   15 +-
 .../apache/spark/sql/hive/CarbonMetastore.scala |2 +-
 .../spark/rdd/CarbonDataRDDFactory.scala|2 +-
 .../sql/CarbonDatasourceHadoopRelation.scala|2 +-
 .../execution/command/AlterTableCommands.scala  |   82 +-
 .../apache/spark/sql/hive/CarbonMetastore.scala |2 +-
 .../org/apache/spark/util/AlterTableUtil.scala  |   82 +-
 .../org/apache/spark/util/Compaction.scala  |2 +-
 .../AlterTableValidationTestCase.scala  |2 +-
 .../merger/AbstractResultProcessor.java |   64 +
 .../merger/CarbonCompactionExecutor.java|  231 +++
 .../processing/merger/CarbonCompactionUtil.java |  383 +
 .../processing/merger/CarbonDataMergerUtil.java | 1385 ++
 .../merger/CarbonDataMergerUtilResult.java  |   33 +
 .../merger/CompactionResultSortProcessor.java   |  401 +
 .../processing/merger/CompactionType.java   |   31 +
 .../processing/merger/NodeBlockRelation.java|   58 +
 .../merger/NodeMultiBlockRelation.java  |   57 +
 .../merger/RowResultMergerProcessor.java|  233 +++
 .../carbondata/processing/merger/TableMeta.java |   40 +
 .../merger/TupleConversionAdapter.java  |   67 +
 .../merger/exeception/SliceMergerException.java |   78 +
 .../newflow/DataLoadProcessBuilder.java |4 +-
 .../steps/DataWriterProcessorStepImpl.java  |   37 +-
 .../sortandgroupby/sortdata/SortParameters.java |3 +-
 .../store/CarbonFactDataHandlerModel.java   |   72 +
 .../util/CarbonDataProcessorUtil.java   |   75 +
 52 files changed, 3427 insertions(+), 2865 deletions(-)
--

[5/7] incubator-carbondata git commit: Compaction lock should also be acquired during alter operation as alter and compaction on same table should not be allowed concurrently.

[ravipesala]

Compaction lock should also be acquired during alter operation as alter and 
compaction on same table should not be allowed concurrently.

Handling for compaction for restructure case. Handled to completely sort the 
data again if any restructured block is selected for compaction

Handled review comments


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/cc59b247
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/cc59b247
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/cc59b247

Branch: refs/heads/master
Commit: cc59b247aff7a6fd5d164d83dfe01890cfcdb2e1
Parents: 0f0907a
Author: manishgupta88 <tomanishgupt...@gmail.com>
Authored: Wed Mar 15 19:24:05 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 16:17:24 2017 +0530

--
 .../core/locks/CarbonLockFactory.java   |8 +
 .../DictionaryBasedVectorResultCollector.java   |   12 +-
 .../RestructureBasedRawResultCollector.java |2 -
 .../spark/compaction/CompactionCallable.java|   44 +
 .../carbondata/spark/load/CarbonLoaderUtil.java |   19 +-
 .../spark/merger/CarbonCompactionExecutor.java  |  211 ---
 .../spark/merger/CarbonCompactionUtil.java  |  354 -
 .../spark/merger/CarbonDataMergerUtil.java  | 1371 -
 .../merger/CarbonDataMergerUtilResult.java  |   33 -
 .../spark/merger/CompactionCallable.java|   44 -
 .../merger/CompactionResultSortProcessor.java   |  563 ---
 .../carbondata/spark/merger/CompactionType.java |   31 -
 .../spark/merger/NodeBlockRelation.java |   58 -
 .../spark/merger/NodeMultiBlockRelation.java|   57 -
 .../spark/merger/RowResultMerger.java   |  367 -
 .../carbondata/spark/merger/TableMeta.java  |   40 -
 .../spark/merger/TupleConversionAdapter.java|   67 -
 .../merger/exeception/SliceMergerException.java |   78 -
 .../spark/rdd/CarbonIUDMergerRDD.scala  |2 +-
 .../carbondata/spark/rdd/CarbonMergerRDD.scala  |   37 +-
 .../apache/carbondata/spark/rdd/Compactor.scala |3 +-
 .../spark/rdd/DataManagementFunc.scala  |3 +-
 .../execution/command/carbonTableSchema.scala   |2 +-
 .../spark/rdd/CarbonDataRDDFactory.scala|2 +-
 .../sql/CarbonDatasourceHadoopRelation.scala|2 +-
 .../spark/sql/CarbonDatasourceRelation.scala|2 +-
 .../sql/execution/command/IUDCommands.scala |   15 +-
 .../apache/spark/sql/hive/CarbonMetastore.scala |2 +-
 .../spark/rdd/CarbonDataRDDFactory.scala|2 +-
 .../sql/CarbonDatasourceHadoopRelation.scala|2 +-
 .../execution/command/AlterTableCommands.scala  |   82 +-
 .../apache/spark/sql/hive/CarbonMetastore.scala |2 +-
 .../org/apache/spark/util/AlterTableUtil.scala  |   82 +-
 .../org/apache/spark/util/Compaction.scala  |2 +-
 .../AlterTableValidationTestCase.scala  |2 +-
 .../merger/AbstractResultProcessor.java |   64 +
 .../merger/CarbonCompactionExecutor.java|  231 +++
 .../processing/merger/CarbonCompactionUtil.java |  383 +
 .../processing/merger/CarbonDataMergerUtil.java | 1385 ++
 .../merger/CarbonDataMergerUtilResult.java  |   33 +
 .../merger/CompactionResultSortProcessor.java   |  401 +
 .../processing/merger/CompactionType.java   |   31 +
 .../processing/merger/NodeBlockRelation.java|   58 +
 .../merger/NodeMultiBlockRelation.java  |   57 +
 .../merger/RowResultMergerProcessor.java|  233 +++
 .../carbondata/processing/merger/TableMeta.java |   40 +
 .../merger/TupleConversionAdapter.java  |   67 +
 .../merger/exeception/SliceMergerException.java |   78 +
 .../newflow/DataLoadProcessBuilder.java |4 +-
 .../steps/DataWriterProcessorStepImpl.java  |   37 +-
 .../sortandgroupby/sortdata/SortParameters.java |3 +-
 .../store/CarbonFactDataHandlerModel.java   |   72 +
 .../util/CarbonDataProcessorUtil.java   |   75 +
 53 files changed, 3427 insertions(+), 3428 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cc59b247/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java 
b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
index 8cc494e..f9520ab 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
@@ -17,6 +17,8 @@
 
 package org.apache.carbondata.core.locks;
 
+import org.apache.carbondata.common.logging.LogServ

[1/7] incubator-carbondata git commit: Compaction lock should also be acquired during alter operation as alter and compaction on same table should not be allowed concurrently.

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 58cd9339e -> 68cbe1508


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cc59b247/processing/src/main/java/org/apache/carbondata/processing/merger/TableMeta.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/merger/TableMeta.java
 
b/processing/src/main/java/org/apache/carbondata/processing/merger/TableMeta.java
new file mode 100644
index 000..b84d695
--- /dev/null
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/merger/TableMeta.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.merger;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+
+public class TableMeta implements Serializable {
+
+  private static final long serialVersionUID = -174987469829431L;
+
+  public CarbonTableIdentifier carbonTableIdentifier;
+  public String storePath;
+  public CarbonTable carbonTable;
+
+  public TableMeta(CarbonTableIdentifier carbonTableIdentifier, String 
storePath,
+  CarbonTable carbonTable) {
+this.carbonTableIdentifier = carbonTableIdentifier;
+this.storePath = storePath;
+this.carbonTable = carbonTable;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cc59b247/processing/src/main/java/org/apache/carbondata/processing/merger/TupleConversionAdapter.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/merger/TupleConversionAdapter.java
 
b/processing/src/main/java/org/apache/carbondata/processing/merger/TupleConversionAdapter.java
new file mode 100644
index 000..f508a87
--- /dev/null
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/merger/TupleConversionAdapter.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.merger;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
+
+/**
+ * This class will be used to convert the Result into the format used in data 
writer.
+ */
+class TupleConversionAdapter {
+
+  private final SegmentProperties segmentproperties;
+
+  private int noDictionaryPresentIndex;
+
+  private int measureCount;
+
+  private boolean isNoDictionaryPresent;
+
+  public TupleConversionAdapter(SegmentProperties segmentProperties) {
+this.measureCount = segmentProperties.getMeasures().size();
+this.isNoDictionaryPresent = 
segmentProperties.getNumberOfNoDictionaryDimension() > 0;
+if (isNoDictionaryPresent) {
+  noDictionaryPresentIndex++;
+}
+this.segmentproperties = segmentProperties;
+  }
+
+  /**
+   * Converting the raw result to the format understandable by the data writer.
+   * @param carbonTuple
+   * @return
+   */
+  public Object[] getObjectArray(Object[] carbonTuple) {
+Object[] row = new Object[measureCount + noDictionaryPresentIndex + 1];
+int index = 0;
+// put measures.
+
+for (int j = 1; j <= measureCount; j++) {
+  row[index++] = carbonTuple[j];
+}
+
+// put No dictionary byte []
+if (isNoDictionaryPresent) {
+  row[index++] = 

[6/7] incubator-carbondata git commit: Added class to handle sorting of data for compaction scenario

[ravipesala]

Added class to handle sorting of data for compaction scenario


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/0f0907a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/0f0907a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/0f0907a2

Branch: refs/heads/master
Commit: 0f0907a26ab51d98e07208426af8af6543aa2cb9
Parents: 58cd933
Author: ravikiran <ravikiran.sn...@gmail.com>
Authored: Wed Mar 15 20:37:26 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 16:17:24 2017 +0530

--
 .../merger/CompactionResultSortProcessor.java   | 563 +++
 1 file changed, 563 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0f0907a2/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CompactionResultSortProcessor.java
--
diff --git 
a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CompactionResultSortProcessor.java
 
b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CompactionResultSortProcessor.java
new file mode 100644
index 000..6cfe8b5
--- /dev/null
+++ 
b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CompactionResultSortProcessor.java
@@ -0,0 +1,563 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.spark.merger;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.model.CarbonLoadModel;
+import org.apache.carbondata.processing.schema.metadata.SortObserver;
+import 
org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
+import org.apache.carbondata.processing.sortandgroupby.sortdata.SortDataRows;
+import 
org.apache.carbondata.processing.sortandgroupby.sortdata.SortIntermediateFileMerger;
+import org.apache.carbondata.processing.sortandgroupby.sortdata.SortParameters;
+import org.apache.carbondata.processing.store.CarbonDataFileAttributes;
+import org.apache.carbondata.processing.store.CarbonFactDataHandlerColumnar;
+import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
+import org.apache.carbondata.processing.store.SingleThreadFinalSortFilesMerger;
+import 
org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+
+import org.pentaho.di.core.exception.KettleException;
+
+/**
+ * This class will process the query result and convert the data
+ * into a format compatible for data load
+ */
+public class CompactionResultSortProcessor {
+
+  /**
+   * LOGGER
+   */
+  private static final LogService LOGGER =
+  
LogServiceFactory.getLogService(CompactionResultSortProcessor.class

[2/2] incubator-carbondata git commit: [CARBONDATA-400] Error message for dataload with a column having more than 100000 characters. This closes #398

[ravipesala]

[CARBONDATA-400] Error message for dataload with a column having more than 
10 characters. This closes #398


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/58cd9339
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/58cd9339
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/58cd9339

Branch: refs/heads/master
Commit: 58cd9339e992a2e5b5d67d714210c0ef59bd099c
Parents: fa7421a c99cf06
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 15:34:24 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 15:34:24 2017 +0530

--
 .../spark/rdd/CarbonGlobalDictionaryRDD.scala   |  3 +++
 .../spark/util/GlobalDictionaryUtil.scala   | 25 +---
 .../processing/csvload/CSVInputFormat.java  |  1 +
 3 files changed, 26 insertions(+), 3 deletions(-)
--

[1/2] incubator-carbondata git commit: [CARBONDATA-400] * Problem: When the number of characters in a column exceeds 100000 characters whole string appears in beeline with exception.

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master fa7421a18 -> 58cd9339e


[CARBONDATA-400]
* Problem: When the number of characters in a column exceeds 10 characters 
whole string appears in beeline with exception.

Analysis: In univocity csv parser settings , the maximum number of characters 
per column is 10 and when it exceeds that limit, TextparsingException is 
thrown with
the complete string as error in beeline during data load.

Fix: Now a proper error message is displayed in beeline and complete error 
messages and parser settings details will be present in logs.
Impact area: Data loading with more than 10 characters in a single column.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c99cf06d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c99cf06d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c99cf06d

Branch: refs/heads/master
Commit: c99cf06de469ac97c3d7269377394d76190554e7
Parents: fa7421a
Author: Akash R Nilugal <akashnilu...@gmail.com>
Authored: Mon Dec 5 15:16:10 2016 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 15:33:53 2017 +0530

--
 .../spark/rdd/CarbonGlobalDictionaryRDD.scala   |  3 +++
 .../spark/util/GlobalDictionaryUtil.scala   | 25 +---
 .../processing/csvload/CSVInputFormat.java  |  1 +
 3 files changed, 26 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c99cf06d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 74531d3..ea71ea1 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -27,6 +27,7 @@ import scala.collection.mutable.ArrayBuffer
 import scala.util.control.Breaks.{break, breakable}
 
 import au.com.bytecode.opencsv.CSVReader
+import com.univocity.parsers.common.TextParsingException
 import org.apache.commons.lang3.{ArrayUtils, StringUtils}
 import org.apache.spark._
 import org.apache.spark.rdd.RDD
@@ -307,6 +308,8 @@ class CarbonBlockDistinctValuesCombineRDD(
   }
   
CarbonTimeStatisticsFactory.getLoadStatisticsInstance.recordLoadCsvfilesToDfTime()
 } catch {
+  case txe: TextParsingException =>
+throw txe
   case ex: Exception =>
 LOGGER.error(ex)
 throw ex

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c99cf06d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 5cb493c..aeb387a 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -32,7 +32,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.io.NullWritable
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-import org.apache.spark.Accumulator
+import org.apache.spark.{Accumulator, SparkException}
 import org.apache.spark.rdd.{NewHadoopRDD, RDD}
 import org.apache.spark.sql._
 import org.apache.spark.sql.types.{StringType, StructField, StructType}
@@ -784,9 +784,28 @@ object GlobalDictionaryUtil {
   }
 } catch {
   case ex: Exception =>
-LOGGER.error(ex, "generate global dictionary failed")
-throw ex
+ex match {
+  case spx: SparkException =>
+LOGGER.error(spx, "generate global dictionary failed")
+throw new Exception("generate global dictionary failed, " +
+trimErrorMessage(spx.getMessage))
+  case _ =>
+LOGGER.error(ex, "generate global dictionary failed")
+throw ex
+}
+}
+  }
+
+  // Get proper error message of TextParsingException
+  def trimErrorMessage(input: String): String = {
+var errorMessage: String = nu

[1/2] incubator-carbondata git commit: fixed issue with meta lock not getting deleted for rename table

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 95549714b -> fa7421a18


fixed issue with meta lock not getting deleted for rename table


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7ae1cd8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7ae1cd8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7ae1cd8f

Branch: refs/heads/master
Commit: 7ae1cd8fe77a391cb5f5addec4e19fbff5b06820
Parents: 9554971
Author: kunal642 <kunal.kap...@knoldus.in>
Authored: Fri Mar 31 16:03:17 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 15:11:22 2017 +0530

--
 .../core/locks/AbstractCarbonLock.java  |  9 +
 .../carbondata/core/locks/HdfsFileLock.java | 36 +---
 .../carbondata/core/locks/ICarbonLock.java  |  8 +
 .../carbondata/core/locks/LocalFileLock.java|  5 ++-
 .../execution/command/AlterTableCommands.scala  | 15 ++--
 .../org/apache/spark/util/AlterTableUtil.scala  |  1 +
 .../AlterTableValidationTestCase.scala  |  8 +
 7 files changed, 60 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
index eeb04d5..e927a7e 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
@@ -18,6 +18,7 @@
 package org.apache.carbondata.core.locks;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.util.CarbonProperties;
 
 /**
@@ -72,4 +73,12 @@ public abstract class AbstractCarbonLock implements 
ICarbonLock {
 
   }
 
+  public boolean releaseLockManually(String lockFile) {
+try {
+  return FileFactory.deleteFile(lockFile, 
FileFactory.getFileType(lockFile));
+} catch (Exception e) {
+  return false;
+}
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
index aed73b3..985ced1 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
@@ -23,6 +23,7 @@ import java.io.IOException;
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
 import org.apache.carbondata.core.util.CarbonProperties;
@@ -102,34 +103,31 @@ public class HdfsFileLock extends AbstractCarbonLock {
*/
   @Override
   public boolean unlock() {
+boolean status = false;
 if (null != dataOutputStream) {
   try {
 dataOutputStream.close();
+status = true;
   } catch (IOException e) {
-try {
-  if (!FileFactory.isFileExist(location, 
FileFactory.getFileType(location))) {
-return true;
-  }
-} catch (IOException e1) {
-  LOGGER.error("Exception in isFileExist of the lock file " + 
e1.getMessage());
-}
-LOGGER.error("Exception in unlocking of the lock file " + 
e.getMessage());
-return false;
+status = false;
   } finally {
-try {
-  if (FileFactory.isFileExist(location, 
FileFactory.getFileType(location))) {
-if (FileFactory.getCarbonFile(location, 
FileFactory.getFileType(location)).delete()) {
-  LOGGER.info("Deleted the lock file " + location);
-} else {
-  LOGGER.error("Not able to delete the lock file " + location);
-}
+CarbonFile carbonFile =
+FileFactory.getCarbonFile(location, 
FileFactory.getFileType(location));
+if (carbonFile.exists()) {
+  if (carbonFile.delete()) {
+LOGGER.info("Deleted the lock file 

[2/2] incubator-carbondata git commit: [CARBONDATA-814] bad record log file writing is not correct. This closes #694

[ravipesala]

[CARBONDATA-814] bad record log file writing is not correct. This closes #694


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/95549714
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/95549714
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/95549714

Branch: refs/heads/master
Commit: 95549714b4f9ea0a244a936b0b5a8ec73ad61b41
Parents: 05b790a 3af2d65
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 14:57:21 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 14:57:21 2017 +0530

--
 .../core/scan/expression/RangeExpressionEvaluator.java   | 7 +++
 .../newflow/steps/DataConverterProcessorStepImpl.java| 8 
 .../processing/store/CarbonFactDataHandlerColumnar.java  | 5 -
 3 files changed, 15 insertions(+), 5 deletions(-)
--

[1/2] incubator-carbondata git commit: if no locktype is configured and store type is HDFS set HDFS lock as default

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master bca8b88bb -> 05b790a35


if no locktype is configured and store type is HDFS set HDFS lock as default


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/41bfd4a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/41bfd4a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/41bfd4a5

Branch: refs/heads/master
Commit: 41bfd4a571c1e63c82c927ee3d487b532a982600
Parents: bca8b88
Author: Manohar <manohar.craz...@gmail.com>
Authored: Wed Apr 5 20:03:30 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 14:09:57 2017 +0530

--
 .../apache/spark/sql/hive/CarbonMetastore.scala | 28 ++--
 .../apache/spark/sql/hive/CarbonMetastore.scala | 28 ++--
 2 files changed, 28 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/41bfd4a5/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
--
diff --git 
a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
 
b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 9fcd185..d5cc540 100644
--- 
a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ 
b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -161,24 +161,24 @@ class CarbonMetastore(hiveContext: HiveContext, val 
storePath: String,
 val statistic = new QueryStatistic()
 // creating zookeeper instance once.
 // if zookeeper is configured as carbon lock type.
-val zookeeperUrl: String = 
hiveContext.getConf(CarbonCommonConstants.ZOOKEEPER_URL, null)
-if (zookeeperUrl != null) {
-  
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL, 
zookeeperUrl)
-  ZookeeperInit.getInstance(zookeeperUrl)
-  LOGGER.info("Zookeeper url is configured. Taking the zookeeper as lock 
type.")
-  var configuredLockType = CarbonProperties.getInstance
-.getProperty(CarbonCommonConstants.LOCK_TYPE)
-  if (null == configuredLockType) {
-configuredLockType = CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER
-CarbonProperties.getInstance
-  .addProperty(CarbonCommonConstants.LOCK_TYPE,
-configuredLockType)
-  }
+val zookeeperurl = 
hiveContext.getConf(CarbonCommonConstants.ZOOKEEPER_URL, null)
+if (null != zookeeperurl) {
+  CarbonProperties.getInstance
+.addProperty(CarbonCommonConstants.ZOOKEEPER_URL, zookeeperurl)
 }
-
 if (metadataPath == null) {
   return null
 }
+// if no locktype is configured and store type is HDFS set HDFS lock as 
default
+if (null == CarbonProperties.getInstance
+  .getProperty(CarbonCommonConstants.LOCK_TYPE) &&
+FileType.HDFS == FileFactory.getFileType(metadataPath)) {
+  CarbonProperties.getInstance
+.addProperty(CarbonCommonConstants.LOCK_TYPE,
+  CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS
+)
+  LOGGER.info("Default lock type HDFSLOCK is configured")
+}
 val fileType = FileFactory.getFileType(metadataPath)
 val metaDataBuffer = new ArrayBuffer[TableMeta]
 fillMetaData(metadataPath, fileType, metaDataBuffer)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/41bfd4a5/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 39d8798..f7ea344 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -200,24 +200,24 @@ class CarbonMetastore(conf: RuntimeConfig, val storePath: 
String) {
 val statistic = new QueryStatistic()
 // creating zookeeper instance once.
 // if zookeeper is configured as carbon lock type.
-val zookeeperUrl: String = conf.get(CarbonCommonConstants.ZOOKEEPER_URL, 
null)
-if (zookeeperUrl != null) {
-  
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL, 
zookeeperUrl)
-  ZookeeperInit.getInstance(zookeeperUrl)
-  LOGGER.info("Zookeeper url is configured. Taking the zookeeper as lock 
type.")
-  var configuredLockType = CarbonProperties.getInstance
-.getProperty(CarbonCommonConst

[2/2] incubator-carbondata git commit: [CARBONDATA-871] If store type is HDFS then set HDFS lock as default. This closes #736

[ravipesala]

[CARBONDATA-871] If store type is HDFS then set HDFS lock as default. This 
closes #736


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/05b790a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/05b790a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/05b790a3

Branch: refs/heads/master
Commit: 05b790a3503942be9a80b80ec4859490321c4658
Parents: bca8b88 41bfd4a
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 14:10:33 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 14:10:33 2017 +0530

--
 .../apache/spark/sql/hive/CarbonMetastore.scala | 28 ++--
 .../apache/spark/sql/hive/CarbonMetastore.scala | 28 ++--
 2 files changed, 28 insertions(+), 28 deletions(-)
--

[1/2] incubator-carbondata git commit: create database DBNAME , here DBNAME should be case insensitive.

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 975a5f7c2 -> bca8b88bb


create database DBNAME , here DBNAME should be case insensitive.

while creating the folder convert to lower case and create.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/0f678331
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/0f678331
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/0f678331

Branch: refs/heads/master
Commit: 0f67833121452c98e57ce9e263fd91bf1cb07557
Parents: 975a5f7
Author: ravikiran <ravikiran.sn...@gmail.com>
Authored: Thu Apr 6 11:06:08 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 14:06:23 2017 +0530

--
 .../src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0f678331/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 3645f95..39d8798 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -657,7 +657,7 @@ class CarbonMetastore(conf: RuntimeConfig, val storePath: 
String) {
   }
 
   def createDatabaseDirectory(dbName: String) {
-val databasePath = storePath + File.separator + dbName
+val databasePath = storePath + File.separator + dbName.toLowerCase
 val fileType = FileFactory.getFileType(databasePath)
 FileFactory.mkdirs(databasePath, fileType)
   }

[2/2] incubator-carbondata git commit: [CARBONDATA-875] in create database DDL , DBNAME should be case insensitive. This closes #742

[ravipesala]

[CARBONDATA-875] in create database DDL , DBNAME should be case insensitive. 
This closes #742


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bca8b88b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bca8b88b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bca8b88b

Branch: refs/heads/master
Commit: bca8b88bba4a056655362ab77411b3a7e301eacf
Parents: 975a5f7 0f67833
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 14:06:51 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 14:06:51 2017 +0530

--
 .../src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--

[1/2] incubator-carbondata git commit: While executing drop table , invalidating the table from hive metastore by running the command externally

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 69edde1d0 -> 975a5f7c2


While executing drop table , invalidating the table from hive metastore by 
running the command externally


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/df3dea90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/df3dea90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/df3dea90

Branch: refs/heads/master
Commit: df3dea90b088e1de2725cbb5cef1c8f90209d9fe
Parents: 69edde1
Author: Manohar 
Authored: Wed Apr 5 20:58:48 2017 +0530
Committer: Manohar 
Committed: Thu Apr 6 12:49:46 2017 +0530

--
 .../carbondata/core/locks/HdfsFileLock.java | 28 
 .../spark/sql/hive/CarbonHiveMetadataUtil.scala |  3 ++-
 2 files changed, 19 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/df3dea90/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
index b3d79e9..aed73b3 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
@@ -23,7 +23,6 @@ import java.io.IOException;
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
 import org.apache.carbondata.core.util.CarbonProperties;
@@ -107,19 +106,26 @@ public class HdfsFileLock extends AbstractCarbonLock {
   try {
 dataOutputStream.close();
   } catch (IOException e) {
+try {
+  if (!FileFactory.isFileExist(location, 
FileFactory.getFileType(location))) {
+return true;
+  }
+} catch (IOException e1) {
+  LOGGER.error("Exception in isFileExist of the lock file " + 
e1.getMessage());
+}
+LOGGER.error("Exception in unlocking of the lock file " + 
e.getMessage());
 return false;
   } finally {
-CarbonFile carbonFile =
-FileFactory.getCarbonFile(location, 
FileFactory.getFileType(location));
-if (carbonFile.exists()) {
-  if (carbonFile.delete()) {
-LOGGER.info("Deleted the lock file " + location);
-  } else {
-LOGGER.error("Not able to delete the lock file " + location);
+try {
+  if (FileFactory.isFileExist(location, 
FileFactory.getFileType(location))) {
+if (FileFactory.getCarbonFile(location, 
FileFactory.getFileType(location)).delete()) {
+  LOGGER.info("Deleted the lock file " + location);
+} else {
+  LOGGER.error("Not able to delete the lock file " + location);
+}
   }
-} else {
-  LOGGER.error("Not able to delete the lock file because "
-  + "it is not existed in location " + location);
+} catch (IOException e) {
+  LOGGER.error("Exception in isFileExist of the lock file " + 
e.getMessage());
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/df3dea90/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
index fc5730b..657d7de 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
@@ -41,7 +41,8 @@ object CarbonHiveMetadataUtil {
   tableName: String,
   sparkSession: SparkSession): Unit = {
 try {
-  sparkSession.sql(s"DROP TABLE IF EXISTS $databaseName.$tableName")
+  
sparkSession.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client.
+runSqlHive(s"DROP TABLE IF EXISTS $databaseName.$tableName")
 } catch {
   case e: Exception =>
 LOGGER.audit(

[2/2] incubator-carbondata git commit: [CARBONDATA-873] Drop table command throwing table already exists exception This closes #739

[ravipesala]

[CARBONDATA-873] Drop table command throwing table already exists exception 
This closes #739


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/975a5f7c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/975a5f7c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/975a5f7c

Branch: refs/heads/master
Commit: 975a5f7c2d0668adbace073b5c7b4bdad8605b80
Parents: 69edde1 df3dea9
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 13:33:51 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 13:33:51 2017 +0530

--
 .../carbondata/core/locks/HdfsFileLock.java | 28 
 .../spark/sql/hive/CarbonHiveMetadataUtil.scala |  3 ++-
 2 files changed, 19 insertions(+), 12 deletions(-)
--

[2/2] incubator-carbondata git commit: [CARBONDATA-863] Support creation and deletion of dictionary files through RDD during alter add and drop This closes #733

[ravipesala]

[CARBONDATA-863] Support creation and deletion of dictionary files through RDD 
during alter add and drop This closes #733


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/65e67918
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/65e67918
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/65e67918

Branch: refs/heads/master
Commit: 65e6791873c019370eed54d164b69db6941e034d
Parents: 4a7adfa b5ba4c6
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 10:54:16 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 10:54:16 2017 +0530

--
 .../core/cache/dictionary/ManageDictionary.java |  85 +++---
 .../spark/rdd/AlterTableAddColumnRDD.scala  | 110 +++
 .../spark/rdd/AlterTableDropColumnRDD.scala |  96 
 .../execution/command/carbonTableSchema.scala   |  21 ++--
 .../execution/command/AlterTableCommands.scala  |  48 +---
 5 files changed, 286 insertions(+), 74 deletions(-)
--

[1/2] incubator-carbondata git commit: Changes done: 1. Clean up of folders created locally during data load and insert into operations. 2. Setting the load status properly for success, partial succes

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master ada081d89 -> 4a7adfa97


Changes done:
1. Clean up of folders created locally during data load and insert into 
operations.
2. Setting the load status properly for success, partial success and failure 
cases.
3. Printing load statistics in case of success and partial success.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/487e41dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/487e41dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/487e41dd

Branch: refs/heads/master
Commit: 487e41ddd13d9e19813cc1b9c4eda73376f1c8ba
Parents: ada081d
Author: manishgupta88 <tomanishgupt...@gmail.com>
Authored: Wed Apr 5 19:04:14 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 10:33:07 2017 +0530

--
 .../carbondata/spark/load/CarbonLoaderUtil.java | 30 
 .../spark/rdd/NewCarbonDataLoadRDD.scala| 30 
 2 files changed, 49 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/487e41dd/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
--
diff --git 
a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
 
b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index cc16398..95f0b10 100644
--- 
a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ 
b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -36,6 +36,9 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -171,13 +174,30 @@ public final class CarbonLoaderUtil {
   tempLocationKey = CarbonCommonConstants.COMPACTION_KEY_WORD + '_' + 
tempLocationKey;
 }
 // form local store location
-String localStoreLocation = CarbonProperties.getInstance()
+final String localStoreLocation = CarbonProperties.getInstance()
 .getProperty(tempLocationKey, 
CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
+// submit local folder clean up in another thread so that main thread 
execution is not blocked
+ExecutorService localFolderDeletionService = 
Executors.newFixedThreadPool(1);
 try {
-  CarbonUtil.deleteFoldersAndFiles(new 
File(localStoreLocation).getParentFile());
-  LOGGER.info("Deleted the local store location" + localStoreLocation);
-} catch (IOException | InterruptedException e) {
-  LOGGER.error(e, "Failed to delete local data load folder location");
+  localFolderDeletionService.submit(new Callable() {
+@Override public Void call() throws Exception {
+  try {
+long startTime = System.currentTimeMillis();
+File file = new File(localStoreLocation);
+CarbonUtil.deleteFoldersAndFiles(file.getParentFile());
+LOGGER.info(
+"Deleted the local store location" + localStoreLocation + " : 
TIme taken: " + (
+System.currentTimeMillis() - startTime));
+  } catch (IOException | InterruptedException e) {
+LOGGER.error(e, "Failed to delete local data load folder 
location");
+  }
+  return null;
+}
+  });
+} finally {
+  if (null != localFolderDeletionService) {
+localFolderDeletionService.shutdown();
+  }
 }
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/487e41dd/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 0690ba1..72ee90f 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -223,7 +223,7 @@ class NewCarbonDataLoadRDD[K, V](
 carbonLoadModel.getTableName + CarbonCommonConstants.UNDERSCORE + 
theSplit.

[2/2] incubator-carbondata git commit: [CARBONDATA-845] Insert Select Into Same Table This closes #723

[ravipesala]

[CARBONDATA-845] Insert Select Into Same Table This closes #723


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ada081d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ada081d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ada081d8

Branch: refs/heads/master
Commit: ada081d89be67ad28c31401acb71a84e3e97244a
Parents: 950a6d0 f527d3d
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Thu Apr 6 10:18:50 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Thu Apr 6 10:18:50 2017 +0530

--
 .../InsertIntoCarbonTableTestCase.scala | 28 +
 .../spark/sql/hive/CarbonAnalysisRules.scala| 43 +++---
 .../spark/sql/CarbonCatalystOperators.scala | 19 +++
 .../sql/CarbonDatasourceHadoopRelation.scala| 13 +
 .../sql/execution/command/DDLStrategy.scala |  5 +-
 .../execution/command/carbonTableSchema.scala   |  3 +-
 .../spark/sql/hive/CarbonAnalysisRules.scala| 60 
 .../spark/sql/hive/CarbonSessionState.scala | 30 +-
 8 files changed, 163 insertions(+), 38 deletions(-)
--

[incubator-carbondata] Git Push Summary

[ravipesala]

Repository: incubator-carbondata
Updated Tags:  refs/tags/apache-carbondata-1.1.0-incubating-rc1 [created] 
647344ff0

incubator-carbondata git commit: [maven-release-plugin] prepare for next development iteration

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/branch-1.1 bf227081b -> 8e50b878f


[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8e50b878
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8e50b878
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8e50b878

Branch: refs/heads/branch-1.1
Commit: 8e50b878f8338306b68a8b735bea37ce71c97b3d
Parents: bf22708
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Wed Apr 5 15:37:41 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 5 15:37:41 2017 +0530

--
 assembly/pom.xml  | 2 +-
 common/pom.xml| 2 +-
 core/pom.xml  | 2 +-
 examples/flink/pom.xml| 2 +-
 examples/spark/pom.xml| 2 +-
 examples/spark2/pom.xml   | 2 +-
 format/pom.xml| 2 +-
 hadoop/pom.xml| 2 +-
 integration/spark-common-test/pom.xml | 2 +-
 integration/spark-common/pom.xml  | 2 +-
 integration/spark/pom.xml | 2 +-
 integration/spark2/pom.xml| 2 +-
 pom.xml   | 4 ++--
 processing/pom.xml| 2 +-
 14 files changed, 15 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8e50b878/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 4d5f2dd..fdcc9b1 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8e50b878/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 71561b0..56ad112 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8e50b878/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index b589328..4ed5b1f 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8e50b878/examples/flink/pom.xml
--
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index 5164315..4b7b4d9 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8e50b878/examples/spark/pom.xml
--
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 10bdb93..6cb5547 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8e50b878/examples/spark2/pom.xml
--
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index d28f606..df0a741 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8e50b878/format/pom.xml
--
diff --git a/format/pom.xml b/format/pom.xml
index a47b45b..a0d69eb 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating
+1.1.1-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8e50b878/hadoop/pom.xml
--
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index a31df27..7b840b9 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml

incubator-carbondata git commit: [maven-release-plugin] prepare release apache-carbondata-1.1.0-incubating-rc1

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/branch-1.1 950a6d0f5 -> bf227081b


[maven-release-plugin] prepare release apache-carbondata-1.1.0-incubating-rc1


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bf227081
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bf227081
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bf227081

Branch: refs/heads/branch-1.1
Commit: bf227081b8c9b0c1eb0839f0f3a49c392baa84d7
Parents: 950a6d0
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Wed Apr 5 15:34:37 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 5 15:34:37 2017 +0530

--
 assembly/pom.xml  | 2 +-
 common/pom.xml| 2 +-
 core/pom.xml  | 2 +-
 examples/flink/pom.xml| 2 +-
 examples/spark/pom.xml| 2 +-
 examples/spark2/pom.xml   | 2 +-
 format/pom.xml| 2 +-
 hadoop/pom.xml| 2 +-
 integration/spark-common-test/pom.xml | 2 +-
 integration/spark-common/pom.xml  | 2 +-
 integration/spark/pom.xml | 2 +-
 integration/spark2/pom.xml| 2 +-
 pom.xml   | 4 ++--
 processing/pom.xml| 2 +-
 14 files changed, 15 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bf227081/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index c1b51f6..4d5f2dd 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bf227081/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 91dcb1f..71561b0 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bf227081/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index 976f8d2..b589328 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bf227081/examples/flink/pom.xml
--
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index d3e8997..5164315 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bf227081/examples/spark/pom.xml
--
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 3b65185..10bdb93 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bf227081/examples/spark2/pom.xml
--
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index 10c59e7..d28f606 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bf227081/format/pom.xml
--
diff --git a/format/pom.xml b/format/pom.xml
index c6c4b2a..a47b45b 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.1.0-incubating-SNAPSHOT
+1.1.0-incubating
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bf227081/hadoop/pom.xml
--
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index e62fa20..a31df27 100644
--- a/hadoop/pom.xml
+++ b

[12/12] incubator-carbondata git commit: [CARBONDATA-823] Refactory data write step This closes #702

[ravipesala]

[CARBONDATA-823] Refactory data write step This closes #702


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/950a6d0f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/950a6d0f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/950a6d0f

Branch: refs/heads/branch-1.1
Commit: 950a6d0f5ab8323aa0feedda48c8c95b323975c8
Parents: bd044c2 8cca0af
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Wed Apr 5 14:37:18 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 5 14:37:18 2017 +0530

--
 .../core/constants/CarbonCommonConstants.java   |   2 +-
 .../core/datastore/NodeMeasureDataStore.java|  32 --
 ...ractHeavyCompressedDoubleArrayDataStore.java |  89 -
 ...yCompressedDoubleArrayDataInMemoryStore.java |  28 --
 .../HeavyCompressedDoubleArrayDataStore.java|  57 
 .../core/util/CarbonMetadataUtil.java   |   2 +-
 .../apache/carbondata/core/util/CarbonUtil.java |   2 +-
 .../carbondata/core/util/DataTypeUtil.java  |   5 +-
 .../sort/unsafe/UnsafeCarbonRowPage.java|   6 +-
 .../holder/UnsafeSortTempFileChunkHolder.java   |   2 +-
 .../merger/UnsafeIntermediateFileMerger.java|   2 +-
 .../sortdata/IntermediateFileMerger.java|   2 +-
 .../sortandgroupby/sortdata/SortDataRows.java   |   2 +-
 .../sortdata/SortTempFileChunkHolder.java   |   2 +-
 .../store/CarbonFactDataHandlerColumnar.java| 328 +++
 .../processing/store/StoreFactory.java  |  33 --
 .../store/writer/CarbonFactDataWriter.java  |   4 +-
 .../writer/v1/CarbonFactDataWriterImplV1.java   |  13 +-
 .../writer/v3/CarbonFactDataWriterImplV3.java   |  14 +-
 19 files changed, 287 insertions(+), 338 deletions(-)
--

[03/12] incubator-carbondata git commit: [CARBONDATA-850] Fix the comment definition issues of CarbonData thrift files

[ravipesala]

[CARBONDATA-850] Fix the comment definition issues of CarbonData thrift files

fix comments


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/970dfd99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/970dfd99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/970dfd99

Branch: refs/heads/branch-1.1
Commit: 970dfd99fbb36940c8feb0de07bed503ec859a41
Parents: 4d0d2d6
Author: chenliang613 <chenliang...@huawei.com>
Authored: Tue Apr 4 15:53:48 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Tue Apr 4 18:06:50 2017 +0530

--
 format/src/main/thrift/carbondata.thrift| 190 ++-
 format/src/main/thrift/carbondata_index.thrift  |  40 ++--
 format/src/main/thrift/dictionary.thrift|  30 ++-
 .../src/main/thrift/dictionary_metadata.thrift  |  30 ++-
 format/src/main/thrift/schema.thrift| 100 +-
 format/src/main/thrift/sort_index.thrift|  32 ++--
 6 files changed, 208 insertions(+), 214 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/970dfd99/format/src/main/thrift/carbondata.thrift
--
diff --git a/format/src/main/thrift/carbondata.thrift 
b/format/src/main/thrift/carbondata.thrift
index 21ff29d..937108c 100644
--- a/format/src/main/thrift/carbondata.thrift
+++ b/format/src/main/thrift/carbondata.thrift
@@ -1,24 +1,22 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
  *
- * http://www.apache.org/licenses/LICENSE-2.0
+ *http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 /**
- * File format description for the carbon file format
+ * File format description for the CarbonData file format
  */
 namespace java org.apache.carbondata.format
 
@@ -26,40 +24,40 @@ include "schema.thrift"
 include "dictionary.thrift"
 
 /**
-* Information about a segment, that represents one data load
-*/
+ * Information about a segment, that represents one data load
+ */
 struct SegmentInfo{
-1: required i32 num_cols; // Number of columns in this load, because 
schema can evolve . TODO: Check whether this is really required
+1: required i32 num_cols; // Number of columns in this load, because 
schema can evolve, different segments may have different columns
 2: required list column_cardinalities; // Cardinality of columns
 }
 
 /**
-*  Btree index of one blocklet
-*/
+ * Btree index of one blocklet
+ */
 struct BlockletBTreeIndex{
 1: required binary start_key; // Bit-packed start key of one blocklet
-2: required binary end_key;// Bit-packed start key of one blocklet
+2: required binary end_key;// Bit-packed end key of one blocklet
 }
 
 /**
-*  Min-max index of one blocklet
-*/
+ * Min-max index of one blocklet
+ */
 struct BlockletMinMaxIndex{
 1: required list min_values; //Min value of all columns of one 
blocklet Bit-Packed
 2: required list max_values; //Max value of all columns of one 
blocklet Bit-Packed
 }
 
 /**
-* Index of one blocklet
-**/
+ * Index of one blocklet
+ */
 struct BlockletIndex{
 1: optional BlockletMinMaxIndex min_max_index;
 2: optional BlockletBTreeIndex b_tree_index

[09/12] incubator-carbondata git commit: Issue : https://issues.apache.org/jira/browse/CARBONDATA-849

[ravipesala]

Issue : https://issues.apache.org/jira/browse/CARBONDATA-849


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8245e5db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8245e5db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8245e5db

Branch: refs/heads/branch-1.1
Commit: 8245e5dbc324e1998fc64e0061018695a49bc088
Parents: c62b299
Author: ravikiran <ravikiran.sn...@gmail.com>
Authored: Tue Apr 4 15:13:25 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 5 13:16:59 2017 +0530

--
 .../apache/spark/sql/execution/command/carbonTableSchema.scala| 2 +-
 .../org/apache/spark/sql/execution/command/DDLStrategy.scala  | 3 ++-
 .../scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala | 3 +++
 3 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8245e5db/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 3346743..6c44264 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -104,7 +104,7 @@ case class AlterTableModel(dbName: Option[String],
segmentUpdateStatusManager: 
Option[SegmentUpdateStatusManager],
compactionType: String,
factTimeStamp: Option[Long],
-   alterSql: String)
+   var alterSql: String)
 
 case class UpdateTableModel(isUpdate: Boolean,
 updatedTimeStamp: Long,

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8245e5db/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
index fb15d6d..2916a9f 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
@@ -72,7 +72,8 @@ class DDLStrategy(sparkSession: SparkSession) extends 
SparkStrategy {
   "Unsupported alter operation on carbon table")
   }
 } else {
-  throw new MalformedCarbonCommandException("Unsupported alter 
operation on hive table")
+  throw new MalformedCarbonCommandException(
+"Operation not allowed : " + altertablemodel.alterSql)
 }
   case 
dataTypeChange@AlterTableDataTypeChange(alterTableChangeDataTypeModel) =>
 val isCarbonTable = CarbonEnv.get.carbonMetastore

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8245e5db/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index dbc3124..695f5fc 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -43,6 +43,9 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
   case x: LoadTable =>
 x.inputSqlString = input
 x
+  case x: AlterTableCompaction =>
+x.alterTableModel.alterSql = input
+x
   case logicalPlan => logicalPlan
 }
 case failureOrError =>

[07/12] incubator-carbondata git commit: CARBONDATA-843 problem: null pointer exception was thrown when floor operation is done on decimal column. analysis: when floor operation was done on decimal co

[ravipesala]

CARBONDATA-843
problem: null pointer exception was thrown when floor operation is done on 
decimal column.
analysis: when floor operation was done on decimal column, scale was greater 
than precision.During floor operation , we tried to change the precision of the 
data.
solution: When sending to the spark layer for performing floor operation, we 
need to send the precision and scale to the spark layer which are taken from 
the CarbonMeasure


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/43319298
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/43319298
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/43319298

Branch: refs/heads/branch-1.1
Commit: 43319298eb122846286e16fdbd145594e4633aa4
Parents: 9e721a8
Author: akash 
Authored: Fri Mar 31 18:10:31 2017 +0530
Committer: jackylk 
Committed: Wed Apr 5 09:46:56 2017 +0800

--
 .../core/scan/collector/impl/AbstractScannedResultCollector.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/43319298/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
index c46d46c..1ecfbc8 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
@@ -106,7 +106,8 @@ public abstract class AbstractScannedResultCollector 
implements ScannedResultCol
 bigDecimalMsrValue =
 bigDecimalMsrValue.setScale(carbonMeasure.getScale(), 
RoundingMode.HALF_UP);
   }
-  return org.apache.spark.sql.types.Decimal.apply(bigDecimalMsrValue);
+  return org.apache.spark.sql.types.Decimal
+  .apply(bigDecimalMsrValue, carbonMeasure.getPrecision(), 
carbonMeasure.getScale());
 default:
   return 
dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
   }

[06/12] incubator-carbondata git commit: makes trivial change for sync up apache git to github

[ravipesala]

makes trivial change for sync up apache git to github


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9e721a81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9e721a81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9e721a81

Branch: refs/heads/branch-1.1
Commit: 9e721a81beb970c72f64c14615d8d5bddbf74de2
Parents: 5c4868e 2c2c7c9
Author: chenliang613 
Authored: Tue Apr 4 20:44:09 2017 +0530
Committer: chenliang613 
Committed: Tue Apr 4 20:44:09 2017 +0530

--
 .../scala/org/apache/carbondata/examples/AllDictionaryExample.scala | 1 +
 1 file changed, 1 insertion(+)
--

[02/12] incubator-carbondata git commit: [CARBONDATA-838]Add decimal column without scale and precision is failing. This closes #713

[ravipesala]

[CARBONDATA-838]Add decimal column without scale and precision is failing. This 
closes #713


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4d0d2d60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4d0d2d60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4d0d2d60

Branch: refs/heads/branch-1.1
Commit: 4d0d2d600fe8271817e43e109878caf51394b888
Parents: 3b62d25 f0fe41b
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Tue Apr 4 16:28:14 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Tue Apr 4 16:28:14 2017 +0530

--
 .../org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala  | 9 ++---
 .../restructure/AlterTableValidationTestCase.scala  | 8 
 2 files changed, 14 insertions(+), 3 deletions(-)
--

[11/12] incubator-carbondata git commit: refactor write step

[ravipesala]

refactor write step


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8cca0afc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8cca0afc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8cca0afc

Branch: refs/heads/branch-1.1
Commit: 8cca0afc5db16557146dfaa33e14c2823d895966
Parents: bd044c2
Author: jackylk <jacky.li...@huawei.com>
Authored: Thu Mar 30 11:21:21 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 5 14:36:47 2017 +0530

--
 .../core/constants/CarbonCommonConstants.java   |   2 +-
 .../core/datastore/NodeMeasureDataStore.java|  32 --
 ...ractHeavyCompressedDoubleArrayDataStore.java |  89 -
 ...yCompressedDoubleArrayDataInMemoryStore.java |  28 --
 .../HeavyCompressedDoubleArrayDataStore.java|  57 
 .../core/util/CarbonMetadataUtil.java   |   2 +-
 .../apache/carbondata/core/util/CarbonUtil.java |   2 +-
 .../carbondata/core/util/DataTypeUtil.java  |   5 +-
 .../sort/unsafe/UnsafeCarbonRowPage.java|   6 +-
 .../holder/UnsafeSortTempFileChunkHolder.java   |   2 +-
 .../merger/UnsafeIntermediateFileMerger.java|   2 +-
 .../sortdata/IntermediateFileMerger.java|   2 +-
 .../sortandgroupby/sortdata/SortDataRows.java   |   2 +-
 .../sortdata/SortTempFileChunkHolder.java   |   2 +-
 .../store/CarbonFactDataHandlerColumnar.java| 328 +++
 .../processing/store/StoreFactory.java  |  33 --
 .../store/writer/CarbonFactDataWriter.java  |   4 +-
 .../writer/v1/CarbonFactDataWriterImplV1.java   |  13 +-
 .../writer/v3/CarbonFactDataWriterImplV3.java   |  14 +-
 19 files changed, 287 insertions(+), 338 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8cca0afc/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 789c321..b82d53c 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -631,7 +631,7 @@ public final class CarbonCommonConstants {
   /**
* DOUBLE_VALUE_MEASURE
*/
-  public static final char SUM_COUNT_VALUE_MEASURE = 'n';
+  public static final char DOUBLE_MEASURE = 'n';
   /**
* BYTE_VALUE_MEASURE
*/

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8cca0afc/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
deleted file mode 100644
index 2f54847..000
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore;
-
-import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
-
-public interface NodeMeasureDataStore {
-  /**
-   * This method will be used to get the writable key array.
-   * writable measure data array will hold below information:
-   * 
-   * total length will be 4 bytes for size + measure data array length
-   *
-   * @return writable array (compressed or normal)
-   */
-  byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] 
dataHolderArray);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8cca0afc/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/AbstractHeavyCom

[08/12] incubator-carbondata git commit: [CARBONDATA-843]null pointer exception was thrown when floor operation is done on decimal column This closes #718

[ravipesala]

[CARBONDATA-843]null pointer exception was thrown when floor operation is done 
on decimal column This closes #718


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c62b2997
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c62b2997
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c62b2997

Branch: refs/heads/branch-1.1
Commit: c62b29979c38b8c30c2169a98b0f204ccae93d07
Parents: 9e721a8 4331929
Author: jackylk 
Authored: Wed Apr 5 09:48:32 2017 +0800
Committer: jackylk 
Committed: Wed Apr 5 09:48:32 2017 +0800

--
 .../core/scan/collector/impl/AbstractScannedResultCollector.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--

[04/12] incubator-carbondata git commit: [CARBONDATA-850] Fix the comment definition issues of CarbonData thrift files This closes #729

[ravipesala]

[CARBONDATA-850] Fix the comment definition issues of CarbonData thrift files 
This closes #729


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5c4868e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5c4868e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5c4868e0

Branch: refs/heads/branch-1.1
Commit: 5c4868e0df07bc0c04c8da45f37d42fcbcd4639f
Parents: 4d0d2d6 970dfd9
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Tue Apr 4 18:07:24 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Tue Apr 4 18:07:24 2017 +0530

--
 format/src/main/thrift/carbondata.thrift| 190 ++-
 format/src/main/thrift/carbondata_index.thrift  |  40 ++--
 format/src/main/thrift/dictionary.thrift|  30 ++-
 .../src/main/thrift/dictionary_metadata.thrift  |  30 ++-
 format/src/main/thrift/schema.thrift| 100 +-
 format/src/main/thrift/sort_index.thrift|  32 ++--
 6 files changed, 208 insertions(+), 214 deletions(-)
--

[10/12] incubator-carbondata git commit: [CARBONDATA-849] Correcting the error message for alter table as per HIVE message for non existing table. This closes #725

[ravipesala]

[CARBONDATA-849] Correcting the error message for alter table as per HIVE 
message for non existing table. This closes #725


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bd044c20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bd044c20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bd044c20

Branch: refs/heads/branch-1.1
Commit: bd044c20895e338884ceeb6ba4bdd5fb15649201
Parents: c62b299 8245e5d
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Wed Apr 5 13:17:31 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 5 13:17:31 2017 +0530

--
 .../apache/spark/sql/execution/command/carbonTableSchema.scala| 2 +-
 .../org/apache/spark/sql/execution/command/DDLStrategy.scala  | 3 ++-
 .../scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala | 3 +++
 3 files changed, 6 insertions(+), 2 deletions(-)
--

[05/12] incubator-carbondata git commit: makes trivial change for sync up apache git to github

[ravipesala]

makes trivial change for sync up apache git to github


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2c2c7c96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2c2c7c96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2c2c7c96

Branch: refs/heads/branch-1.1
Commit: 2c2c7c9660e1deef13b9aae71d94f2a439340ed7
Parents: 5c4868e
Author: chenliang613 
Authored: Tue Apr 4 20:42:44 2017 +0530
Committer: chenliang613 
Committed: Tue Apr 4 20:42:44 2017 +0530

--
 .../scala/org/apache/carbondata/examples/AllDictionaryExample.scala | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2c2c7c96/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
--
diff --git 
a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
 
b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
index df5ee01..9012dcf 100644
--- 
a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
+++ 
b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.carbondata.examples
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants

[01/12] incubator-carbondata git commit: add decimal column without scale and precision is failing. [Forced Update!]

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/branch-1.1 d3bb59749 -> 950a6d0f5 (forced update)


add decimal column without scale and precision is failing.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f0fe41b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f0fe41b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f0fe41b9

Branch: refs/heads/branch-1.1
Commit: f0fe41b9cc1b547062ede9557d6e397e1ce38974
Parents: 3b62d25
Author: nareshpr <prnaresh.nar...@gmail.com>
Authored: Thu Mar 30 16:52:51 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Tue Apr 4 16:27:34 2017 +0530

--
 .../org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala  | 9 ++---
 .../restructure/AlterTableValidationTestCase.scala  | 8 
 2 files changed, 14 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f0fe41b9/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 8120942..9d2c245 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -873,9 +873,12 @@ abstract class CarbonDDLSqlParser extends 
AbstractCarbonSparkSQLParser {
* Matching the decimal(10,0) data type and returning the same.
*/
   private lazy val decimalType =
-  DECIMAL ~ ("(" ~> numericLit <~ ",") ~ (numericLit <~ ")") ^^ {
-case decimal ~ precision ~ scale =>
-  s"$decimal($precision, $scale)"
+  DECIMAL ~ (("(" ~> numericLit <~ ",") ~ (numericLit <~ ")")).? ^^ {
+case decimal ~ precisionAndScale => if (precisionAndScale.isDefined) {
+  s"$decimal(${ precisionAndScale.get._1 }, ${ precisionAndScale.get._2 })"
+} else {
+  s"$decimal(10,0)"
+}
   }
 
   protected lazy val nestedType: Parser[Field] = structFieldType | 
arrayFieldType |

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f0fe41b9/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
index 0a59497..bd34913 100644
--- 
a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
@@ -104,6 +104,14 @@ class AlterTableValidationTestCase extends QueryTest with 
BeforeAndAfterAll {
 checkExistence(sql("desc restructure"), true, "dcmldecimal(5,4)")
   }
 
+  test(
+"test add decimal without scale and precision, default precision and scale 
(10,0) should be " +
+"used")
+  {
+sql("alter table restructure add columns(dcmldefault decimal)")
+checkExistence(sql("desc restructure"), true, "dcmldefaultdecimal(10,0)")
+  }
+
   test("test adding existing measure as dimension") {
 sql("alter table restructure add columns(dcmlfld decimal(5,4))")
 try {

[1/2] incubator-carbondata git commit: refactor write step

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master bd044c208 -> 950a6d0f5


refactor write step


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8cca0afc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8cca0afc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8cca0afc

Branch: refs/heads/master
Commit: 8cca0afc5db16557146dfaa33e14c2823d895966
Parents: bd044c2
Author: jackylk <jacky.li...@huawei.com>
Authored: Thu Mar 30 11:21:21 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 5 14:36:47 2017 +0530

--
 .../core/constants/CarbonCommonConstants.java   |   2 +-
 .../core/datastore/NodeMeasureDataStore.java|  32 --
 ...ractHeavyCompressedDoubleArrayDataStore.java |  89 -
 ...yCompressedDoubleArrayDataInMemoryStore.java |  28 --
 .../HeavyCompressedDoubleArrayDataStore.java|  57 
 .../core/util/CarbonMetadataUtil.java   |   2 +-
 .../apache/carbondata/core/util/CarbonUtil.java |   2 +-
 .../carbondata/core/util/DataTypeUtil.java  |   5 +-
 .../sort/unsafe/UnsafeCarbonRowPage.java|   6 +-
 .../holder/UnsafeSortTempFileChunkHolder.java   |   2 +-
 .../merger/UnsafeIntermediateFileMerger.java|   2 +-
 .../sortdata/IntermediateFileMerger.java|   2 +-
 .../sortandgroupby/sortdata/SortDataRows.java   |   2 +-
 .../sortdata/SortTempFileChunkHolder.java   |   2 +-
 .../store/CarbonFactDataHandlerColumnar.java| 328 +++
 .../processing/store/StoreFactory.java  |  33 --
 .../store/writer/CarbonFactDataWriter.java  |   4 +-
 .../writer/v1/CarbonFactDataWriterImplV1.java   |  13 +-
 .../writer/v3/CarbonFactDataWriterImplV3.java   |  14 +-
 19 files changed, 287 insertions(+), 338 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8cca0afc/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 789c321..b82d53c 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -631,7 +631,7 @@ public final class CarbonCommonConstants {
   /**
* DOUBLE_VALUE_MEASURE
*/
-  public static final char SUM_COUNT_VALUE_MEASURE = 'n';
+  public static final char DOUBLE_MEASURE = 'n';
   /**
* BYTE_VALUE_MEASURE
*/

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8cca0afc/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
deleted file mode 100644
index 2f54847..000
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore;
-
-import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
-
-public interface NodeMeasureDataStore {
-  /**
-   * This method will be used to get the writable key array.
-   * writable measure data array will hold below information:
-   * 
-   * total length will be 4 bytes for size + measure data array length
-   *
-   * @return writable array (compressed or normal)
-   */
-  byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] 
dataHolderArray);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8cca0afc/core/src/main/java/org/apache/

[2/2] incubator-carbondata git commit: [CARBONDATA-823] Refactory data write step This closes #702

[ravipesala]

[CARBONDATA-823] Refactory data write step This closes #702


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/950a6d0f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/950a6d0f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/950a6d0f

Branch: refs/heads/master
Commit: 950a6d0f5ab8323aa0feedda48c8c95b323975c8
Parents: bd044c2 8cca0af
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Wed Apr 5 14:37:18 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 5 14:37:18 2017 +0530

--
 .../core/constants/CarbonCommonConstants.java   |   2 +-
 .../core/datastore/NodeMeasureDataStore.java|  32 --
 ...ractHeavyCompressedDoubleArrayDataStore.java |  89 -
 ...yCompressedDoubleArrayDataInMemoryStore.java |  28 --
 .../HeavyCompressedDoubleArrayDataStore.java|  57 
 .../core/util/CarbonMetadataUtil.java   |   2 +-
 .../apache/carbondata/core/util/CarbonUtil.java |   2 +-
 .../carbondata/core/util/DataTypeUtil.java  |   5 +-
 .../sort/unsafe/UnsafeCarbonRowPage.java|   6 +-
 .../holder/UnsafeSortTempFileChunkHolder.java   |   2 +-
 .../merger/UnsafeIntermediateFileMerger.java|   2 +-
 .../sortdata/IntermediateFileMerger.java|   2 +-
 .../sortandgroupby/sortdata/SortDataRows.java   |   2 +-
 .../sortdata/SortTempFileChunkHolder.java   |   2 +-
 .../store/CarbonFactDataHandlerColumnar.java| 328 +++
 .../processing/store/StoreFactory.java  |  33 --
 .../store/writer/CarbonFactDataWriter.java  |   4 +-
 .../writer/v1/CarbonFactDataWriterImplV1.java   |  13 +-
 .../writer/v3/CarbonFactDataWriterImplV3.java   |  14 +-
 19 files changed, 287 insertions(+), 338 deletions(-)
--

[1/2] incubator-carbondata git commit: Issue : https://issues.apache.org/jira/browse/CARBONDATA-849

[ravipesala]

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master c62b29979 -> bd044c208


Issue : https://issues.apache.org/jira/browse/CARBONDATA-849


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8245e5db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8245e5db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8245e5db

Branch: refs/heads/master
Commit: 8245e5dbc324e1998fc64e0061018695a49bc088
Parents: c62b299
Author: ravikiran <ravikiran.sn...@gmail.com>
Authored: Tue Apr 4 15:13:25 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 5 13:16:59 2017 +0530

--
 .../apache/spark/sql/execution/command/carbonTableSchema.scala| 2 +-
 .../org/apache/spark/sql/execution/command/DDLStrategy.scala  | 3 ++-
 .../scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala | 3 +++
 3 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8245e5db/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 3346743..6c44264 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -104,7 +104,7 @@ case class AlterTableModel(dbName: Option[String],
segmentUpdateStatusManager: 
Option[SegmentUpdateStatusManager],
compactionType: String,
factTimeStamp: Option[Long],
-   alterSql: String)
+   var alterSql: String)
 
 case class UpdateTableModel(isUpdate: Boolean,
 updatedTimeStamp: Long,

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8245e5db/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
index fb15d6d..2916a9f 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
@@ -72,7 +72,8 @@ class DDLStrategy(sparkSession: SparkSession) extends 
SparkStrategy {
   "Unsupported alter operation on carbon table")
   }
 } else {
-  throw new MalformedCarbonCommandException("Unsupported alter 
operation on hive table")
+  throw new MalformedCarbonCommandException(
+"Operation not allowed : " + altertablemodel.alterSql)
 }
   case 
dataTypeChange@AlterTableDataTypeChange(alterTableChangeDataTypeModel) =>
 val isCarbonTable = CarbonEnv.get.carbonMetastore

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8245e5db/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index dbc3124..695f5fc 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -43,6 +43,9 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
   case x: LoadTable =>
 x.inputSqlString = input
 x
+  case x: AlterTableCompaction =>
+x.alterTableModel.alterSql = input
+x
   case logicalPlan => logicalPlan
 }
 case failureOrError =>

[2/2] incubator-carbondata git commit: [CARBONDATA-849] Correcting the error message for alter table as per HIVE message for non existing table. This closes #725

[ravipesala]

[CARBONDATA-849] Correcting the error message for alter table as per HIVE 
message for non existing table. This closes #725


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bd044c20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bd044c20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bd044c20

Branch: refs/heads/master
Commit: bd044c20895e338884ceeb6ba4bdd5fb15649201
Parents: c62b299 8245e5d
Author: ravipesala <ravi.pes...@gmail.com>
Authored: Wed Apr 5 13:17:31 2017 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Wed Apr 5 13:17:31 2017 +0530

--
 .../apache/spark/sql/execution/command/carbonTableSchema.scala| 2 +-
 .../org/apache/spark/sql/execution/command/DDLStrategy.scala  | 3 ++-
 .../scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala | 3 +++
 3 files changed, 6 insertions(+), 2 deletions(-)
--