[47/51] [abbrv] carbondata git commit: [CARBONDATA-1425] Inappropriate Exception displays while creating a new partition with incorrect partition type

[ravipesala]

[CARBONDATA-1425] Inappropriate Exception displays while creating a new 
partition with incorrect partition type

Change the error content when the range info data mismatch the partition 
field's data type, the new showing content as below:Data in range info must be 
the same type with the partition field's type

This closes #1336


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7e5e29c7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7e5e29c7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7e5e29c7

Branch: refs/heads/branch-1.2
Commit: 7e5e29c740e1b1a253379ed969b260accc73b09f
Parents: d3cb6f6
Author: mayun 
Authored: Wed Sep 6 22:52:39 2017 +0800
Committer: chenliang613 
Committed: Mon Sep 18 14:48:25 2017 +0800

--
 .../carbondata/spark/util/CommonUtil.scala  |   7 +-
 .../partition/TestAlterPartitionTable.scala | 118 ++-
 2 files changed, 123 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/7e5e29c7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index ed4d784..fd265a8 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -334,12 +334,17 @@ object CommonUtil {
 timestampFormatter, dateFormatter)
 }
 val iterator = rangeInfo.tail.toIterator
-while(iterator.hasNext) {
+while (iterator.hasNext) {
   val next = columnDataType match {
 case DataType.STRING => ByteUtil.toBytes(iterator.next())
 case _ => PartitionUtil.getDataBasedOnDataType(iterator.next(), 
columnDataType,
   timestampFormatter, dateFormatter)
   }
+  if (next == null) {
+sys.error(
+  "Data in range info must be the same type with the partition field's 
type "
++ columnDataType)
+  }
   if (comparator.compare(head, next) < 0) {
 head = next
   } else {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/7e5e29c7/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index 9de2ef5..0c59bd9 100644
--- 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -36,7 +36,8 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 
   override def beforeAll {
 dropTable
-
+CarbonProperties.getInstance()
+  .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "-MM-dd")
 CarbonProperties.getInstance()
   .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "/MM/dd")
 /**
@@ -569,6 +570,111 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 checkAnswer(result_after5, result_origin5)
   }
 
+   test("test exception when alter partition and the values"
+   + "in range_info can not match partition column type") {
+ val exception_test_range_int: Exception = intercept[Exception] {
+  sql(
+"""
+  | CREATE TABLE test_range_int(col1 INT, col2 STRING)
+  | PARTITIONED BY (col3 INT) STORED BY 'carbondata'
+  | TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='11,12')
+""".stripMargin)
+   sql("ALTER TABLE test_range_int ADD PARTITION ('abc')")
+}
+ assert(exception_test_range_int.getMessage
+ .contains("Data in range info must be the same type with the 
partition field's type"))
+
+sql("DROP TABLE IF EXISTS test_range_smallint")
+val exception_test_range_smallint: Exception = intercept[Exception] {
+  sql(
+"""
+  | CREATE TABLE test_range_smallint(col1 INT, col2 STRING)
+  | PARTITIONED BY (col3 SMALLINT) STORED BY 'carbondata'
+  | TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='11,12')
+""".stripMargin)
+  sql("ALTER TABLE test_range_smallint ADD PARTITION ('abc')")
+}
+  

[34/51] [abbrv] carbondata git commit: [CARBONDATA-1412] - Fixed bug related to incorrect behavior of delete functionality while using segment.starttime before ''

[ravipesala]

[CARBONDATA-1412] - Fixed bug related to incorrect behavior of delete 
functionality while using segment.starttime before ''

This closes #1316


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/df95547d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/df95547d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/df95547d

Branch: refs/heads/branch-1.2
Commit: df95547d1b7e79c04407966a4f07ba6dbd6442da
Parents: c15a11d
Author: SangeetaGulia 
Authored: Mon Sep 4 12:34:54 2017 +0530
Committer: Ravindra Pesala 
Committed: Thu Sep 14 14:05:49 2017 +0530

--
 .../carbondata/core/statusmanager/LoadMetadataDetails.java   | 2 +-
 .../org/apache/carbondata/hadoop/test/util/StoreCreator.java | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/df95547d/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
 
b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index c2acedb..3f83c72 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -52,7 +52,7 @@ public class LoadMetadataDetails implements Serializable {
 
   // dont remove static as the write will fail.
   private static final SimpleDateFormat parser =
-  new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
+  new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
   /**
* Segment modification or deletion time stamp
*/

http://git-wip-us.apache.org/repos/asf/carbondata/blob/df95547d/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
--
diff --git 
a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java 
b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index beca50d..9be3ed5 100644
--- 
a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ 
b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -144,7 +144,7 @@ public class StoreCreator {
   loadModel.setDateFormat(null);
   
loadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
   CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
-  CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+  CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS));
   
loadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(
   CarbonCommonConstants.CARBON_DATE_FORMAT,
   CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT));
@@ -495,7 +495,7 @@ public class StoreCreator {
   }
 
   public static String readCurrentTime() {
-SimpleDateFormat sdf = new 
SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
+SimpleDateFormat sdf = new 
SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
 String date = null;
 
 date = sdf.format(new Date());

[02/51] [abbrv] carbondata git commit: [CARBONDATA-1433] Added Vectorized Reader for Presto Integration

[ravipesala]

[CARBONDATA-1433] Added Vectorized Reader for Presto Integration

This PR is for optimizing the Presto Integration Performance. 1)Added 
Vectorized Reader for reading the data 2)Used DictionaryBlock for loading the 
dictionary values. 3) Removed unused code

This closes #1307


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/531dcd23
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/531dcd23
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/531dcd23

Branch: refs/heads/branch-1.2
Commit: 531dcd23457add78ad397a00129ba7efb01a0228
Parents: 0c519c4
Author: Bhavya 
Authored: Tue Aug 29 17:02:18 2017 +0530
Committer: chenliang613 
Committed: Thu Sep 7 21:07:29 2017 +0800

--
 integration/presto/pom.xml  |  43 +++
 .../carbondata/presto/CarbonTypeUtil.java   |  34 +++
 .../presto/CarbonVectorizedRecordReader.java| 264 +++
 .../carbondata/presto/CarbondataPageSource.java | 256 ++
 .../presto/CarbondataRecordCursor.java  |  30 ++-
 .../carbondata/presto/CarbondataRecordSet.java  |  40 ++-
 .../presto/CarbondataRecordSetProvider.java |  11 +-
 .../presto/CarbondataSplitManager.java  | 181 +
 .../presto/ColumnarVectorWrapper.java   | 209 +++
 .../presto/readers/AbstractStreamReader.java|  66 +
 .../readers/DecimalSliceStreamReader.java   | 183 +
 .../presto/readers/DoubleStreamReader.java  |  71 +
 .../presto/readers/IntegerStreamReader.java |  67 +
 .../presto/readers/LongStreamReader.java|  62 +
 .../presto/readers/ObjectStreamReader.java  |  73 +
 .../presto/readers/SliceStreamReader.java   | 107 
 .../carbondata/presto/readers/StreamReader.java |  42 +++
 .../presto/readers/StreamReaders.java   |  67 +
 .../CarbonDictionaryDecodeReadSupport.scala | 144 ++
 .../presto/CarbonDictionaryDecodeSupport.scala  |  66 -
 20 files changed, 1625 insertions(+), 391 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/531dcd23/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 3cddc1e..562718f 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -46,8 +46,15 @@
   org.apache.carbondata
   carbondata-core
   ${project.version}
+  
+
+  org.apache.spark
+  spark-sql_2.10
+
+  
 
 
+
 
   org.apache.carbondata
   carbondata-common
@@ -58,6 +65,12 @@
   org.apache.carbondata
   carbondata-processing
   ${project.version}
+  
+
+  org.apache.spark
+  spark-sql_2.10
+
+  
 
 
 
@@ -139,6 +152,36 @@
   hadoop-apache2
   2.7.3-1
 
+
+
+  org.apache.spark
+  spark-core_2.11
+  2.1.0
+  
+
+  com.fasterxml.jackson.core
+  jackson-databind
+
+  
+
+
+
+  org.apache.spark
+  spark-catalyst_2.11
+  2.1.0
+
+
+
+  org.apache.spark
+  spark-sql_2.11
+  2.1.0
+  
+
+  com.fasterxml.jackson.core
+  jackson-databind
+
+  
+
   
 
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/531dcd23/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java
--
diff --git 
a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java
 
b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java
new file mode 100644
index 000..6cb2915
--- /dev/null
+++ 
b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java
@@ -0,0 +1,34 @@
+package org.apache.carbondata.presto;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+import org.apache.spark.sql.types.DataTypes;
+
+public class CarbonTypeUtil {
+
+  public static org.apache.spark.sql.types.DataType 
convertCarbonToSparkDataType(
+  DataType carbonDataType) {
+switch (carbonDataType) {
+  case STRING:
+return DataTypes.StringType;
+  case SHORT:
+return DataTypes.ShortType;
+  case INT:
+return DataTypes.IntegerType;
+  case LONG:
+return DataTypes.LongType;
+  case DOUBLE:
+return DataTypes.DoubleType;
+  case BOOLEAN:
+return DataTypes.BooleanType;
+  case DECIMAL:
+return DataTypes.createDecimalType();
+  case TIMESTAMP:
+return DataTypes.TimestampType;
+  case DATE:
+

[27/51] [abbrv] carbondata git commit: [CARBONDATA-1117]updated configuration-parameters.md and useful-tips-on-carbondata.md files for SET/RESET

[ravipesala]

[CARBONDATA-1117]updated configuration-parameters.md and 
useful-tips-on-carbondata.md files for SET/RESET

This closes #1219


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b414393b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b414393b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b414393b

Branch: refs/heads/branch-1.2
Commit: b414393bed319ea970036aac72f140b9389bccf9
Parents: 2176a2f
Author: vandana 
Authored: Mon Jul 31 19:32:58 2017 +0530
Committer: Jacky Li 
Committed: Wed Sep 13 19:24:35 2017 +0800

--
 docs/configuration-parameters.md  | 87 +-
 docs/useful-tips-on-carbondata.md |  1 -
 2 files changed, 85 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b414393b/docs/configuration-parameters.md
--
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index bdd551a..8101aa1 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -24,6 +24,7 @@
  * [Performance Configuration](#performance-configuration)
  * [Miscellaneous Configuration](#miscellaneous-configuration)
  * [Spark Configuration](#spark-configuration)
+ * [Dynamic Configuration In CarbonData Using 
SET-RESET](#dynamic-configuration-in-carbondata-using-set-reset)
  
  
 ##  System Configuration
@@ -146,5 +147,87 @@ This section provides the details of all the 
configurations required for CarbonD
 
|||-|
 | spark.driver.memory | 1g | Amount of memory to be used by the driver 
process. |
 | spark.executor.memory | 1g | Amount of memory to be used per executor 
process. |
-   
- 
+
+##  Dynamic Configuration In CarbonData Using SET-RESET
+
+**SET/RESET** commands are used to add, update, display, or reset the 
carbondata properties dynamically without restarting the driver.
+
+**Syntax**
+
+* **Add or Update :** This command adds or updates the value of parameter_name.
+
+```
+SET parameter_name=parameter_value
+```
+
+* Display Property Value: This command displays the value of the specified 
parameter_name.
+
+```
+SET parameter_name
+```
+
+* Display Session Parameters: This command displays all the supported session 
parameters.
+
+```
+SET
+```
+
+* Display Session Parameters along with usage details: This command displays 
all the supported session parameters along with their usage details.
+
+```
+SET -v
+```
+
+* Reset: This command clears all the session parameters.
+
+```
+RESET
+```
+
+ **Parameter Description:**
+
+| Parameter   | Description
|
+|-||
+| parameter_name  | Name of the property whose value needs to be dynamically 
added, updated, or displayed. |
+| parameter_value | New value of the parameter_name to be set. 
|
+
+Dynamically Configurable Properties of CarbonData
+
+| Properties   | Description   






   |

[17/51] [abbrv] carbondata git commit: [CARBONDATA-1470] csv data should not show in error log when data column length is greater than 100000 characters

[ravipesala]

[CARBONDATA-1470] csv data should not show in error log when data column length 
is greater than 10 characters

Added method to handle TextParsingException and stop the leak of the sensitive 
data in logs

This closes #1349


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/33ecca9b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/33ecca9b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/33ecca9b

Branch: refs/heads/branch-1.2
Commit: 33ecca9b7c256285b2f10a080b1c6bae14fef5a3
Parents: 1551a7c
Author: kapilreja 
Authored: Tue Sep 12 06:32:36 2017 +
Committer: Ravindra Pesala 
Committed: Tue Sep 12 15:34:31 2017 +0530

--
 .../processing/csvload/CSVRecordReaderIterator.java |  8 
 .../processing/util/CarbonDataProcessorUtil.java| 16 
 2 files changed, 24 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/33ecca9b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
 
b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
index 10a036a..efe75ef 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
@@ -19,8 +19,12 @@ package org.apache.carbondata.processing.csvload;
 
 import java.io.IOException;
 
+
 import org.apache.carbondata.common.CarbonIterator;
 import 
org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+
+import com.univocity.parsers.common.TextParsingException;
 
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -62,6 +66,10 @@ public class CSVRecordReaderIterator extends 
CarbonIterator {
   }
   return true;
 } catch (Exception e) {
+  if (e instanceof TextParsingException) {
+throw new CarbonDataLoadingException(
+CarbonDataProcessorUtil.trimErrorMessage(e.getMessage()));
+  }
   throw new CarbonDataLoadingException(e);
 }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/33ecca9b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
 
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 45461e3..e91cf44 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -610,4 +610,20 @@ public final class CarbonDataProcessorUtil {
 }
 return outArr;
   }
+
+  /**
+   * This method returns String if exception is TextParsingException
+   *
+   * @param input
+   * @return
+   */
+  public static String trimErrorMessage(String input) {
+String errorMessage = input;
+if (input != null) {
+  if (input.split("Hint").length > 0) {
+errorMessage = input.split("Hint")[0];
+  }
+}
+return errorMessage;
+  }
 }
\ No newline at end of file

[29/51] [abbrv] carbondata git commit: [CARBONDATA-1400] Fix bug of array column out of bound when writing carbondata file

[ravipesala]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8c1ddbf2/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
new file mode 100644
index 000..f08444b
--- /dev/null
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.encoding;
+
+import org.apache.carbondata.core.datastore.TableSpec;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import 
org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveDeltaIntegralCodec;
+import 
org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveFloatingCodec;
+import 
org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveIntegralCodec;
+import 
org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec;
+import 
org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.ComplexDimensionIndexCodec;
+import 
org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.DictDimensionIndexCodec;
+import 
org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.DirectDictDimensionIndexCodec;
+import 
org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.HighCardDictDimensionIndexCodec;
+import org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+/**
+ * Default factory will select encoding base on column page data type and 
statistics
+ */
+public class DefaultEncodingFactory extends EncodingFactory {
+
+  private static final int THREE_BYTES_MAX = (int) Math.pow(2, 23) - 1;
+  private static final int THREE_BYTES_MIN = - THREE_BYTES_MAX - 1;
+
+  private static final boolean newWay = false;
+
+  private static EncodingFactory encodingFactory = new 
DefaultEncodingFactory();
+
+  public static EncodingFactory getInstance() {
+// TODO: make it configurable after added new encodingFactory
+return encodingFactory;
+  }
+
+  @Override
+  public ColumnPageEncoder createEncoder(TableSpec.ColumnSpec columnSpec, 
ColumnPage inputPage) {
+// TODO: add log
+if (columnSpec instanceof TableSpec.MeasureSpec) {
+  return createEncoderForMeasure(inputPage);
+} else {
+  if (newWay) {
+return createEncoderForDimension((TableSpec.DimensionSpec) columnSpec, 
inputPage);
+  } else {
+assert columnSpec instanceof TableSpec.DimensionSpec;
+return createEncoderForDimensionLegacy((TableSpec.DimensionSpec) 
columnSpec);
+  }
+}
+  }
+
+  private ColumnPageEncoder createEncoderForDimension(TableSpec.DimensionSpec 
columnSpec,
+  ColumnPage inputPage) {
+Compressor compressor = CompressorFactory.getInstance().getCompressor();
+switch (columnSpec.getColumnType()) {
+  case GLOBAL_DICTIONARY:
+  case DIRECT_DICTIONARY:
+  case PLAIN_VALUE:
+return new 
DirectCompressCodec(inputPage.getDataType()).createEncoder(null);
+  case COMPLEX:
+return new ComplexDimensionIndexCodec(false, false, 
compressor).createEncoder(null);
+  default:
+throw new RuntimeException("unsupported dimension type: " +
+columnSpec.getColumnType());
+}
+  }
+
+  private ColumnPageEncoder 
createEncoderForDimensionLegacy(TableSpec.DimensionSpec columnSpec) {
+TableSpec.DimensionSpec dimensionSpec = columnSpec;
+Compressor compressor = CompressorFactory.getInstance().getCompressor();
+switch (dimensionSpec.getColumnType()) {
+  case GLOBAL_DICTIONARY:
+return new DictDimensionIndexCodec(
+dimensionSpec.isInSortColumns(),
+dimensionSpec.isInSortColumns() && 

[50/51] [abbrv] carbondata git commit: [CARBONDATA-1316] Support drop partition function

[ravipesala]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/cb51b862/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index 0c59bd9..3646fad 100644
--- 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -260,8 +260,8 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 checkAnswer(result_after, result_origin)
 
 val result_after1 = sql(s"select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area where area < 'OutSpace' ")
-val rssult_origin1 = sql(s"select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area_origin where area < 'OutSpace' ")
-checkAnswer(result_after1, rssult_origin1)
+val result_origin1 = sql(s"select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area_origin where area < 'OutSpace' ")
+checkAnswer(result_after1, result_origin1)
 
 val result_after2 = sql("select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area where area <= 'OutSpace' ")
 val result_origin2 = sql("select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area_origin where area <= 'OutSpace' ")
@@ -279,28 +279,24 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 val result_origin5 = sql("select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area_origin where area >= 'OutSpace' ")
 checkAnswer(result_after5, result_origin5)
 
-sql("""ALTER TABLE list_table_area ADD PARTITION ('One', '(Two, Three)', 
'Four')""".stripMargin)
-val carbonTable1 = 
CarbonMetadata.getInstance().getCarbonTable("default_list_table_area")
-val partitionInfo1 = 
carbonTable1.getPartitionInfo(carbonTable.getFactTableName)
-val partitionIds1 = partitionInfo1.getPartitionIds
-val new_list_info = partitionInfo1.getListInfo
-assert(partitionIds1 == List(0, 1, 2, 3, 4, 5, 6, 7, 
8).map(Integer.valueOf(_)).asJava)
-assert(partitionInfo1.getMAX_PARTITION == 8)
-assert(partitionInfo1.getNumPartitions == 9)
-assert(new_list_info.get(0).get(0) == "Asia")
-assert(new_list_info.get(1).get(0) == "America")
-assert(new_list_info.get(2).get(0) == "Europe")
-assert(new_list_info.get(3).get(0) == "OutSpace")
-assert(new_list_info.get(4).get(0) == "Hi")
-assert(new_list_info.get(5).get(0) == "One")
-assert(new_list_info.get(6).get(0) == "Two")
-assert(new_list_info.get(6).get(1) == "Three")
-assert(new_list_info.get(7).get(0) == "Four")
-validateDataFiles("default_list_table_area", "0", Seq(0, 1, 2, 4))
-
-val result_after6 = sql("select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area")
-val result_origin6 = sql("""select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area_origin""")
-checkAnswer(result_after6, result_origin6)
+intercept[Exception]  { sql("""ALTER TABLE DROP PARTITION(0)""")}
+intercept[Exception]  { sql("""ALTER TABLE DROP PARTITION(0) WITH DATA""")}
+
+sql("""ALTER TABLE list_table_area DROP PARTITION(2) WITH DATA""")
+val carbonTable2 = 
CarbonMetadata.getInstance().getCarbonTable("default_list_table_area")
+val partitionInfo2 = 
carbonTable2.getPartitionInfo(carbonTable.getFactTableName)
+val partitionIds2 = partitionInfo2.getPartitionIds
+val list_info2 = partitionInfo2.getListInfo
+assert(partitionIds2 == List(0, 1, 3, 4, 5).map(Integer.valueOf(_)).asJava)
+assert(partitionInfo2.getMAX_PARTITION == 5)
+assert(partitionInfo2.getNumPartitions == 5)
+assert(list_info2.get(0).get(0) == "Asia")
+assert(list_info2.get(1).get(0) == "Europe")
+assert(list_info2.get(2).get(0) == "OutSpace")
+assert(list_info2.get(3).get(0) == "Hi")
+validateDataFiles("default_list_table_area", "0", Seq(0, 1, 4))
+checkAnswer(sql("select id, vin, logdate, phonenumber, country, area, 
salary from list_table_area"),
+  sql("select id, vin, logdate, phonenumber, country, area, salary from 
list_table_area_origin where area <> 'America' "))
   }
 
   test("Alter table add partition: Range Partition") {
@@ -309,9 +305,9 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 val partitionInfo = 
carbonTable.getPartitionInfo(carbonTable.getFactTableName)
 val partitionIds = partitionInfo.getPartitionIds
 val range_info = partitionInfo.getRangeInfo
-

[10/51] [abbrv] carbondata git commit: [CARBONDATA-1451] Removing configuration for number_of_rows_per_blocklet_column_page

[ravipesala]

[CARBONDATA-1451] Removing configuration for 
number_of_rows_per_blocklet_column_page

This closes #1334


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/435ea26e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/435ea26e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/435ea26e

Branch: refs/heads/branch-1.2
Commit: 435ea26eb8864db44b2e246d7f47a416d2dfdbd4
Parents: a5483e8
Author: dhatchayani 
Authored: Wed Sep 6 15:25:33 2017 +0530
Committer: Ravindra Pesala 
Committed: Sat Sep 9 18:45:57 2017 +0530

--
 .../constants/CarbonV3DataFormatConstants.java  | 18 +--
 .../carbondata/core/scan/filter/FilterUtil.java |  3 +-
 .../scan/scanner/AbstractBlockletScanner.java   | 12 +++-
 .../carbondata/core/util/CarbonProperties.java  | 32 
 .../store/CarbonFactDataHandlerColumnar.java|  4 +--
 5 files changed, 8 insertions(+), 61 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/435ea26e/core/src/main/java/org/apache/carbondata/core/constants/CarbonV3DataFormatConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonV3DataFormatConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonV3DataFormatConstants.java
index edc7b9a..e888986 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonV3DataFormatConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonV3DataFormatConstants.java
@@ -61,24 +61,8 @@ public interface CarbonV3DataFormatConstants {
   short NUMBER_OF_COLUMN_TO_READ_IN_IO_MIN = 1;
 
   /**
-   * number of rows per blocklet column page
-   */
-  @CarbonProperty
-  String NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE = 
"number.of.rows.per.blocklet.column.page";
-
-  /**
* number of rows per blocklet column page default value
*/
-  String NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT = "32000";
-
-  /**
-   * number of rows per blocklet column page max value
-   */
-  short NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_MAX = 32000;
-
-  /**
-   * number of rows per blocklet column page min value
-   */
-  short NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_MIN = 8000;
+  short NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT = 32000;
 
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/435ea26e/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 78c1afd..01e1cfa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -1602,8 +1602,7 @@ public final class FilterUtil {
   public static BitSetGroup createBitSetGroupWithDefaultValue(int pageCount, 
int totalRowCount,
   boolean defaultValue) {
 BitSetGroup bitSetGroup = new BitSetGroup(pageCount);
-int numberOfRows = Integer
-
.parseInt(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
+int numberOfRows = 
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
 int pagesTobeFullFilled = totalRowCount / numberOfRows;
 int rowCountForLastPage = totalRowCount % numberOfRows;
 for (int i = 0; i < pagesTobeFullFilled; i++) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/435ea26e/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
index 0e1ede8..1e4becd 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
@@ -32,17 +32,12 @@ import 
org.apache.carbondata.core.scan.result.impl.NonFilterQueryScannedResult;
 import org.apache.carbondata.core.stats.QueryStatistic;
 import org.apache.carbondata.core.stats.QueryStatisticsConstants;
 import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.util.CarbonProperties;
 
 /**
  * Blocklet scanner class to process the block
  */
 public abstract class AbstractBlockletScanner implements BlockletScanner {
 
-  private static final int NUMBER_OF_ROWS_PER_PAGE = 

[48/51] [abbrv] carbondata git commit: [CARBONDATA-1477] Corrected the Bug For Wrong Values Of Date data type in hive

[ravipesala]

[CARBONDATA-1477] Corrected the Bug For Wrong Values Of Date data type in hive

This closes #1356


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c2574688
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c2574688
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c2574688

Branch: refs/heads/branch-1.2
Commit: c2574688f82d4bd9bfea6e33d04d2ae1f96ea0f1
Parents: 7e5e29c
Author: anubhav100 
Authored: Wed Sep 13 17:50:19 2017 +0530
Committer: chenliang613 
Committed: Mon Sep 18 16:06:36 2017 +0800

--
 .../carbondata/hive/CarbonDictionaryDecodeReadSupport.java  | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/c2574688/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
--
diff --git 
a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
 
b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
index 76597fd..34a1936 100644
--- 
a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
+++ 
b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.sql.Date;
 import java.sql.Timestamp;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.List;
 
 import org.apache.carbondata.core.cache.Cache;
@@ -228,9 +229,13 @@ public class CarbonDictionaryDecodeReadSupport 
implements CarbonReadSupport

[16/51] [abbrv] carbondata git commit: [CARBONDATA-1469] Optimizations for Presto Integration

[ravipesala]

[CARBONDATA-1469] Optimizations for Presto Integration

This closes #1348


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1551a7c7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1551a7c7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1551a7c7

Branch: refs/heads/branch-1.2
Commit: 1551a7c7d4046964a299d01a927b2900a84dc2f3
Parents: 0ab928e
Author: Bhavya 
Authored: Mon Sep 11 16:33:07 2017 +0530
Committer: CHEN LIANG 
Committed: Tue Sep 12 07:08:37 2017 +0800

--
 integration/presto/pom.xml  | 536 ---
 .../carbondata/presto/PrestoFilterUtil.java |  75 ++-
 .../readers/DecimalSliceStreamReader.java   |  58 +-
 .../presto/readers/DoubleStreamReader.java  |  27 +-
 .../presto/readers/IntegerStreamReader.java |  28 +-
 .../presto/readers/LongStreamReader.java|  27 +-
 .../presto/readers/ShortStreamReader.java   |  80 +++
 .../presto/readers/StreamReaders.java   |   6 +
 .../presto/readers/TimestampStreamReader.java   |  79 +++
 9 files changed, 682 insertions(+), 234 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/1551a7c7/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 562718f..617ce93 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -15,7 +15,9 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 -->
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 
   4.0.0
 
@@ -37,49 +39,223 @@
 
   
 
-  org.apache.thrift
-  libthrift
-  0.9.3
-
-
-
   org.apache.carbondata
-  carbondata-core
+  carbondata-hadoop
   ${project.version}
   
 
   org.apache.spark
-  spark-sql_2.10
+  spark-network-shuffle_2.11
+
+
+  org.apache.spark
+  spark-sketch_2.11
+
+
+  org.slf4j
+  slf4j-log4j12
+
+
+  net.java.dev.jets3t
+  jets3t
+
+
+  javax.servlet
+  javax.servlet-api
+
+
+  org.apache.commons
+  commons-math3
+
+
+  org.apache.httpcomponents
+  httpclient
+
+
+  org.antlr
+  antlr4-runtime
+
+
+  com.esotericsoftware
+  minlog
+
+
+  org.codehaus.janino
+  janino
+
+
+  net.jpountz.lz4
+  lz4
+
+
+  net.sf.py4j
+  py4j
+
+
+  org.spark-project.spark
+  unused
+
+
+  org.apache.hadoop
+  hadoop-common
+
+
+  org.apache.hadoop
+  hadoop-client
+
+
+  org.apache.hadoop
+  hadoop-hdfs
+
+
+  org.apache.spark
+  spark-tags_2.11
+
+
+  org.apache.parquet
+  parquet-column
+
+
+  org.apache.parquet
+  parquet-hadoop
+
+
+  org.glassfish.jersey.core
+  jersey-client
+
+
+  org.glassfish.jersey.core
+  jersey-common
+
+
+  org.glassfish.jersey.core
+  jersey-server
+
+
+  org.glassfish.jersey.containers
+  jersey-container-servlet
+
+
+  org.glassfish.jersey.containers
+  jersey-container-servlet-core
+
+
+  org.glassfish.jersey.containers
+  jersey-container-servlet-core
 
-  
-
-
-
-
-  org.apache.carbondata
-  carbondata-common
-  ${project.version}
-
 
-
-  org.apache.carbondata
-  carbondata-processing
-  ${project.version}
-  
+
+  org.apache.curator
+  curator-recipes
+
+
+  org.apache.avro
+  avro-mapred
+
+
+  com.twitter
+  chill_2.11
+
+
+  io.dropwizard.metrics
+  metrics-core
+
+
+  io.dropwizard.metrics
+  metrics-jvm
+
+
+  

[40/51] [abbrv] carbondata git commit: [CARBONDATA-1429] Add a value based compression for decimal data type when decimal is stored as Int or Long

[ravipesala]

[CARBONDATA-1429] Add a value based compression for decimal data type when 
decimal is stored as Int or Long

Added a value based compression for decimal data type when decimal is stored as 
Int or Long

When decimal precision is <= 9, decimal values are stored in 4 bytes but they 
are not compressed further based on min and max values as compared with other 
primitive data type compression. Therefore now based on min and max value 
decimal data falling in Integer range will be further compressed as byte or 
short.
When decimal precision is <= 18, decimal values are stored in 8 bytes but they 
are not compressed further based on min and max values as compared with other 
primitive data type compression. Therefore now based on min and max value 
decimal data falling in Long range will be further compressed as byte, short or 
int.
Advantage: This will reduce the storage space thereby decreasing the IO time 
while decompressing the data.

This closes #1297


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6f204376
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6f204376
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6f204376

Branch: refs/heads/branch-1.2
Commit: 6f204376f880231c8f537052fe1b29008178aad8
Parents: 302ef2f
Author: manishgupta88 
Authored: Thu Aug 24 12:43:58 2017 +0530
Committer: Jacky Li 
Committed: Fri Sep 15 17:22:58 2017 +0800

--
 .../core/datastore/page/ColumnPage.java |  64 +++-
 .../core/datastore/page/DecimalColumnPage.java  | 109 +++
 .../core/datastore/page/LazyColumnPage.java |  20 +-
 .../datastore/page/SafeDecimalColumnPage.java   | 227 ++
 .../datastore/page/SafeFixLengthColumnPage.java |   3 +-
 .../datastore/page/SafeVarLengthColumnPage.java |   5 +-
 .../datastore/page/UnsafeDecimalColumnPage.java | 296 +++
 .../page/UnsafeVarLengthColumnPage.java |  52 +---
 .../datastore/page/VarLengthColumnPageBase.java |  52 +++-
 .../page/encoding/ColumnPageEncoderMeta.java|  16 +-
 .../page/encoding/DefaultEncodingFactory.java   |  93 +-
 .../adaptive/AdaptiveDeltaIntegralCodec.java|  21 +-
 .../adaptive/AdaptiveIntegralCodec.java |  12 +-
 .../datatype/DecimalConverterFactory.java   |  74 +++--
 14 files changed, 931 insertions(+), 113 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f204376/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 0be409e..6c534d6 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -22,7 +22,6 @@ import java.math.BigDecimal;
 import java.util.BitSet;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.ColumnType;
 import org.apache.carbondata.core.datastore.TableSpec;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
@@ -31,7 +30,6 @@ import 
org.apache.carbondata.core.datastore.page.statistics.ColumnPageStatsColle
 import org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult;
 import org.apache.carbondata.core.memory.MemoryException;
 import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DecimalConverterFactory;
 import org.apache.carbondata.core.util.CarbonProperties;
 
 import static org.apache.carbondata.core.metadata.datatype.DataType.BYTE;
@@ -61,8 +59,6 @@ public abstract class ColumnPage {
   // statistics collector for this column page
   private ColumnPageStatsCollector statsCollector;
 
-  DecimalConverterFactory.DecimalConverter decimalConverter;
-
   protected static final boolean unsafe = 
Boolean.parseBoolean(CarbonProperties.getInstance()
   .getProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING,
   CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING_DEFAULT));
@@ -75,12 +71,6 @@ public abstract class ColumnPage {
 this.dataType = dataType;
 this.pageSize = pageSize;
 this.nullBitSet = new BitSet(pageSize);
-if (dataType == DECIMAL) {
-  assert (columnSpec.getColumnType() == ColumnType.MEASURE);
-  int precision = columnSpec.getPrecision();
-  int scale = columnSpec.getScale();
-  decimalConverter = 
DecimalConverterFactory.INSTANCE.getDecimalConverter(precision, scale);
-   

[01/51] [abbrv] carbondata git commit: [CARBONDATA-1433] Added Vectorized Reader for Presto Integration [Forced Update!]

[ravipesala]

Repository: carbondata
Updated Branches:
  refs/heads/branch-1.2 fe8867250 -> cb51b8621 (forced update)


http://git-wip-us.apache.org/repos/asf/carbondata/blob/531dcd23/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala
--
diff --git 
a/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala
 
b/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala
new file mode 100644
index 000..e3985e0
--- /dev/null
+++ 
b/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.presto
+
+import com.facebook.presto.spi.block.SliceArrayBlock
+import io.airlift.slice.{Slice, Slices}
+import io.airlift.slice.Slices._
+
+import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
+import org.apache.carbondata.core.cache.dictionary.{Dictionary, 
DictionaryChunksWrapper,
+DictionaryColumnUniqueIdentifier}
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.datatype.DataType
+import org.apache.carbondata.core.metadata.encoder.Encoding
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
+import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport
+
+/**
+ * This is the class to decode dictionary encoded column data back to its 
original value.
+ */
+class CarbonDictionaryDecodeReadSupport[T] extends CarbonReadSupport[T] {
+  private var dictionaries: Array[Dictionary] = _
+  private var dataTypes: Array[DataType] = _
+  private var dictionarySliceArray: Array[SliceArrayBlock] = _
+
+  /**
+   * This initialization is done inside executor task
+   * for column dictionary involved in decoding.
+   *
+   * @param carbonColumns   column list
+   * @param absoluteTableIdentifier table identifier
+   */
+
+  override def initialize(carbonColumns: Array[CarbonColumn],
+  absoluteTableIdentifier: AbsoluteTableIdentifier) {
+
+dictionaries = new Array[Dictionary](carbonColumns.length)
+dataTypes = new Array[DataType](carbonColumns.length)
+dictionarySliceArray = new Array[SliceArrayBlock](carbonColumns.length)
+
+carbonColumns.zipWithIndex.foreach {
+  case (carbonColumn, index) => if 
(carbonColumn.hasEncoding(Encoding.DICTIONARY) &&
+
!carbonColumn.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
+!carbonColumn.isComplex) {
+val cacheProvider: CacheProvider = CacheProvider.getInstance
+val forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, 
Dictionary] =
+  cacheProvider
+.createCache(CacheType.FORWARD_DICTIONARY, 
absoluteTableIdentifier.getStorePath)
+dataTypes(index) = carbonColumn.getDataType
+dictionaries(index) = forwardDictionaryCache
+  .get(new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier
+.getCarbonTableIdentifier, carbonColumn.getColumnIdentifier))
+dictionarySliceArray(index) = 
createSliceArrayBlock(dictionaries(index))
+
+  }
+  else {
+dataTypes(index) = carbonColumn.getDataType
+  }
+}
+
+  }
+
+  /**
+   * Function to create the SliceArrayBlock with dictionary Data
+   *
+   * @param dictionaryData
+   * @return
+   */
+  private def createSliceArrayBlock(dictionaryData: Dictionary): 
SliceArrayBlock = {
+val chunks: DictionaryChunksWrapper = dictionaryData.getDictionaryChunks
+val sliceArray = new Array[Slice](chunks.getSize + 1)
+// Initialize Slice Array with Empty Slice as per Presto's code
+sliceArray(0) = (Slices.EMPTY_SLICE)
+var count = 1
+while (chunks.hasNext) {
+  {
+val value: Array[Byte] = chunks.next
+if (count == 1) {
+  sliceArray(count + 1) = null
+}
+else {
+  sliceArray(count) = wrappedBuffer(value, 0, 

[36/51] [abbrv] carbondata git commit: [SDV]Disable tests in other modules except cluster

[ravipesala]

[SDV]Disable tests in other modules except cluster

This closes #1358


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/642b4bf7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/642b4bf7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/642b4bf7

Branch: refs/heads/branch-1.2
Commit: 642b4bf738a91b7d744d6ce619028efd0bef103f
Parents: b269384
Author: Ravindra Pesala 
Authored: Thu Sep 14 09:14:21 2017 +0530
Committer: chenliang613 
Committed: Thu Sep 14 20:25:04 2017 +0800

--
 .../filter/executer/RangeValueFilterExecuterImpl.java |  2 +-
 hadoop/pom.xml|  9 -
 integration/hive/pom.xml  |  8 
 integration/presto/pom.xml|  8 
 .../cluster/sdv/generated/DataLoadingV3TestCase.scala |  4 ++--
 .../cluster/sdv/generated/GlobalSortTestCase.scala|  2 +-
 .../cluster/sdv/generated/PartitionTestCase.scala |  6 +++---
 .../cluster/sdv/generated/QueriesNormalTestCase.scala |  6 --
 .../cluster/sdv/generated/V3offheapvectorTestCase.scala   |  4 ++--
 .../apache/carbondata/cluster/sdv/suite/SDVSuites.scala   | 10 +-
 10 files changed, 42 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/642b4bf7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 63472f9..f2d5a69 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -552,7 +552,7 @@ public class RangeValueFilterExecuterImpl extends 
ValueBasedFilterExecuterImpl {
   if 
(dimColEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
 DirectDictionaryGenerator directDictionaryGenerator = 
DirectDictionaryKeyGeneratorFactory
 
.getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType());
-int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
+int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 
1;
 CarbonDimension currentBlockDimension =
 segmentProperties.getDimensions().get(dimensionBlocksIndex);
 if (currentBlockDimension.isSortColumn()) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/642b4bf7/hadoop/pom.xml
--
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index aa7992a..4df8922 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -65,5 +65,12 @@
   
 
   
-
+  
+
+  sdvtest
+  
+true
+  
+
+  
 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/642b4bf7/integration/hive/pom.xml
--
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index b122c04..17a3cad 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -182,5 +182,13 @@
 
 
 
+
+
+sdvtest
+
+true
+
+
+
 
 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/642b4bf7/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 924a2be..b23b1be 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -581,4 +581,12 @@
   
 
   
+  
+
+  sdvtest
+  
+true
+  
+
+  
 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/642b4bf7/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingV3TestCase.scala
--
diff --git 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingV3TestCase.scala
 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingV3TestCase.scala
index 3389c2e..1c98832 100644
--- 

[46/51] [abbrv] carbondata git commit: [CARBONDATA-1438] Unify the sort column and sort scope in create table command

[ravipesala]

[CARBONDATA-1438] Unify the sort column and sort scope in create table command

In order to improve the ease of usage for users, unify the sort column and sort 
scope in create table command.

This closes #1321


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d3cb6f65
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d3cb6f65
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d3cb6f65

Branch: refs/heads/branch-1.2
Commit: d3cb6f653b087cd69c347ed16208bc7bac3af9a7
Parents: dd42277
Author: chenerlu 
Authored: Mon Sep 4 20:54:55 2017 +0800
Committer: Jacky Li 
Committed: Sun Sep 17 18:03:13 2017 +0800

--
 .../sdv/generated/BatchSortLoad1TestCase.scala  |  24 ++--
 .../sdv/generated/BatchSortLoad2TestCase.scala  |  16 +--
 .../sdv/generated/BatchSortLoad3TestCase.scala  |  30 +---
 .../sdv/generated/BatchSortQueryTestCase.scala  |   8 +-
 .../sdv/generated/GlobalSortTestCase.scala  | 142 +--
 .../TestCreateTableWithSortScope.scala  | 134 +
 .../dataload/TestBatchSortDataLoad.scala|  48 ++-
 .../dataload/TestGlobalSortDataLoad.scala   |  88 +---
 .../testsuite/sortcolumns/TestSortColumns.scala |  11 +-
 .../carbondata/spark/load/ValidateUtil.scala|   2 +-
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |  12 +-
 .../execution/command/carbonTableSchema.scala   |  35 -
 .../execution/command/carbonTableSchema.scala   |  28 +++-
 13 files changed, 337 insertions(+), 241 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3cb6f65/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad1TestCase.scala
--
diff --git 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad1TestCase.scala
 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad1TestCase.scala
index aeeab57..9eb5dec 100644
--- 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad1TestCase.scala
+++ 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad1TestCase.scala
@@ -243,15 +243,15 @@ class BatchSortLoad1TestCase extends QueryTest with 
BeforeAndAfterAll {
   }
 
 
-  //To check incremental load one with batch_sort and others configured with 
different sort
+  //To check incremental load one with batch_sort
   test("Batch_sort_Loading_001-01-01-01_001-TC_021", Include) {
- sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME 
String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 
bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 
decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 
int) STORED BY 'carbondata'""").collect
-   sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' 
into table uniqdata20a OPTIONS('DELIMITER'=',' 
,'SORT_SCOPE'='LOCAL_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
-   sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' 
into table uniqdata20a OPTIONS('DELIMITER'=',' 
,'SORT_SCOPE'='BATCH_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
-   sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' 
into table uniqdata20a OPTIONS('DELIMITER'=',' 
,'SORT_SCOPE'='NO_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
-   sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' 
into table uniqdata20a OPTIONS('DELIMITER'=',' 

[14/51] [abbrv] carbondata git commit: [CARBONDATA-649] fix for update with rand function

[ravipesala]

[CARBONDATA-649] fix for update with rand function

This closes #1296


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8b38e0b3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8b38e0b3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8b38e0b3

Branch: refs/heads/branch-1.2
Commit: 8b38e0b3844d2220d6639d25bcafbab7a7af75f7
Parents: 590bbb9
Author: ashwini-krishnakumar 
Authored: Thu Sep 7 07:36:32 2017 +
Committer: Ravindra Pesala 
Committed: Mon Sep 11 14:07:09 2017 +0530

--
 .../iud/UpdateCarbonTableTestCase.scala | 30 +++
 .../sql/CustomDeterministicExpression.scala | 41 +++
 .../spark/sql/hive/CarbonStrategies.scala   | 52 ++
 .../spark/sql/optimizer/CarbonOptimizer.scala   | 55 
 .../sql/CustomDeterministicExpression.scala | 42 +++
 .../execution/CarbonLateDecodeStrategy.scala| 49 +
 .../sql/optimizer/CarbonLateDecodeRule.scala| 43 +--
 7 files changed, 251 insertions(+), 61 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/8b38e0b3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
index 623416b..4186fa2 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
@@ -448,6 +448,36 @@ class UpdateCarbonTableTestCase extends QueryTest with 
BeforeAndAfterAll {
 sql("DROP TABLE IF EXISTS default.carbon1")
   }
 
+  test("update table in carbondata with rand() ") {
+
+sql("""CREATE TABLE iud.rand(imei string,age int,task bigint,num 
double,level decimal(10,3),name string)STORED BY 'org.apache.carbondata.format' 
""")
+sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/update01.csv' INTO 
TABLE iud.rand OPTIONS('DELIMITER'=',' , 
'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='imei,age,task,num,level,name')""").collect
+
+sql("select substring(name,1,2 ) , name ,getTupleId() as tupleId , rand()  
from  iud.rand").show(100)
+
+sql("select name , substring(name,1,2 ) ,getTupleId() as tupleId , num , 
rand() from  iud.rand").show(100)
+
+sql("Update  rand set (num) = (rand())").show()
+
+sql("Update  rand set (num) = (rand(9))").show()
+
+sql("Update  rand set (name) = ('Lily')").show()
+
+sql("select name ,  num from  iud.rand").show(100)
+
+sql("select  imei , age , name , num  from  iud.rand").show(100)
+
+sql("select rand() , getTupleId() as tupleId from  iud.rand").show(100)
+
+sql("select * from  iud.rand").show(100)
+
+sql("select  imei , rand() , num from  iud.rand").show(100)
+
+sql("select  name , rand()  from  iud.rand").show(100)
+
+sql("DROP TABLE IF EXISTS iud.rand")
+  }
+
   override def afterAll {
 sql("use default")
 sql("drop database  if exists iud cascade")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8b38e0b3/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
--
diff --git 
a/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
 
b/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
new file mode 100644
index 000..d745be2
--- /dev/null
+++ 
b/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing 

[11/51] [abbrv] carbondata git commit: [CARBONDATA-1420] Fixed bug for creation of partitioned table with date datatype column

[ravipesala]

[CARBONDATA-1420] Fixed bug for creation of partitioned table with date 
datatype column

Added code to fetch the default date and timestamp format if not specified in 
carbon.properties.
Added exception case when partition values are not compatible with datatype of 
partition column.
Added test case for fetching default timestamp and date formats in case of 
create partition table command.

This closes #1319


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/252c3e33
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/252c3e33
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/252c3e33

Branch: refs/heads/branch-1.2
Commit: 252c3e335e69fc0342de825c38a01b51cf0330a9
Parents: 435ea26
Author: Geetika Gupta 
Authored: Mon Sep 4 16:51:18 2017 +0530
Committer: Ravindra Pesala 
Committed: Sat Sep 9 18:53:52 2017 +0530

--
 .../partition/TestDDLForPartitionTable.scala|  32 +--
 ...ForPartitionTableWithDefaultProperties.scala | 207 +++
 .../carbondata/spark/util/CommonUtil.scala  |  15 +-
 3 files changed, 235 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/252c3e33/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
index 561ebf5..8083fde 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
@@ -197,7 +197,7 @@ class TestDDLForPartitionTable  extends QueryTest with 
BeforeAndAfterAll {
   | TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='abc,def')
 """.stripMargin)
 }
-assert(exception_test_list_int.getMessage.contains("Invalid partition 
definition"))
+assert(exception_test_list_int.getMessage.contains("Invalid Partition 
Values"))
 
 sql("DROP TABLE IF EXISTS test_list_small")
 val exception_test_list_small: Exception = intercept[Exception] {
@@ -208,7 +208,7 @@ class TestDDLForPartitionTable  extends QueryTest with 
BeforeAndAfterAll {
   | TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='abc,def')
 """.stripMargin)
 }
-assert(exception_test_list_small.getMessage.contains("Invalid partition 
definition"))
+assert(exception_test_list_small.getMessage.contains("Invalid Partition 
Values"))
 
 sql("DROP TABLE IF EXISTS test_list_float")
 val exception_test_list_float: Exception = intercept[Exception] {
@@ -219,7 +219,7 @@ class TestDDLForPartitionTable  extends QueryTest with 
BeforeAndAfterAll {
   | TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='abc,def')
 """.stripMargin)
 }
-assert(exception_test_list_float.getMessage.contains("Invalid partition 
definition"))
+assert(exception_test_list_float.getMessage.contains("Invalid Partition 
Values"))
 
 sql("DROP TABLE IF EXISTS test_list_double")
 val exception_test_list_double: Exception = intercept[Exception] {
@@ -230,7 +230,7 @@ class TestDDLForPartitionTable  extends QueryTest with 
BeforeAndAfterAll {
   | TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='abc,def')
 """.stripMargin)
 }
-assert(exception_test_list_double.getMessage.contains("Invalid partition 
definition"))
+assert(exception_test_list_double.getMessage.contains("Invalid Partition 
Values"))
 
 sql("DROP TABLE IF EXISTS test_list_bigint")
 val exception_test_list_bigint: Exception = intercept[Exception] {
@@ -241,7 +241,7 @@ class TestDDLForPartitionTable  extends QueryTest with 
BeforeAndAfterAll {
   | TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='abc,def')
 """.stripMargin)
 }
-assert(exception_test_list_bigint.getMessage.contains("Invalid partition 
definition"))
+assert(exception_test_list_bigint.getMessage.contains("Invalid Partition 
Values"))
 
 sql("DROP TABLE IF EXISTS test_list_date")
 val exception_test_list_date: Exception = intercept[Exception] {
@@ -252,7 +252,7 @@ class TestDDLForPartitionTable  extends QueryTest with 
BeforeAndAfterAll {
   | TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='abc,def')
 """.stripMargin)
 }
-assert(exception_test_list_date.getMessage.contains("Invalid partition 

[25/51] [abbrv] carbondata git commit: [CARBONDATA-1471] Replace BigDecimal to double to improve performance

[ravipesala]

[CARBONDATA-1471] Replace BigDecimal to double to improve performance

While calculating adaptive floating encoding currently it uses BigDecimal for 
calculations, But it is very slow to use BIgdecimal as it creates many objects 
of BigDecimals. Alternatively, we can use double to improve the performance.

This closes #1345


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a8b3face
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a8b3face
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a8b3face

Branch: refs/heads/branch-1.2
Commit: a8b3face6271562d415922af737e3e9b22d2fce0
Parents: b6727d7
Author: Ravindra Pesala 
Authored: Sun Sep 10 14:21:15 2017 +0530
Committer: Jacky Li 
Committed: Wed Sep 13 16:45:58 2017 +0800

--
 .../adaptive/AdaptiveFloatingCodec.java | 47 
 1 file changed, 18 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8b3face/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
index 7fc5811..789383c 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
@@ -18,7 +18,6 @@
 package org.apache.carbondata.core.datastore.page.encoding.adaptive;
 
 import java.io.IOException;
-import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -45,7 +44,7 @@ import org.apache.carbondata.format.Encoding;
 public class AdaptiveFloatingCodec extends AdaptiveCodec {
 
   private ColumnPage encodedPage;
-  private BigDecimal factor;
+  private Double factor;
 
   public static ColumnPageCodec newInstance(DataType srcDataType, DataType 
targetDataType,
   SimpleStatsResult stats) {
@@ -55,7 +54,7 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
   public AdaptiveFloatingCodec(DataType srcDataType, DataType targetDataType,
   SimpleStatsResult stats) {
 super(srcDataType, targetDataType, stats);
-this.factor = BigDecimal.valueOf(Math.pow(10, stats.getDecimalCount()));
+this.factor = Math.pow(10, stats.getDecimalCount());
   }
 
   @Override
@@ -98,8 +97,8 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
   public ColumnPageDecoder createDecoder(ColumnPageEncoderMeta meta) {
 assert meta instanceof AdaptiveFloatingEncoderMeta;
 AdaptiveFloatingEncoderMeta codecMeta = (AdaptiveFloatingEncoderMeta) meta;
-final Compressor compressor = 
CompressorFactory.getInstance().getCompressor(
-codecMeta.getCompressorName());
+final Compressor compressor =
+
CompressorFactory.getInstance().getCompressor(codecMeta.getCompressorName());
 final DataType targetDataType = codecMeta.getTargetDataType();
 return new ColumnPageDecoder() {
   @Override
@@ -141,24 +140,19 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
 public void encode(int rowId, float value) {
   switch (targetDataType) {
 case BYTE:
-  encodedPage.putByte(rowId,
-  BigDecimal.valueOf(value).multiply(factor).byteValue());
+  encodedPage.putByte(rowId, (byte) (value * factor));
   break;
 case SHORT:
-  encodedPage.putShort(rowId,
-  BigDecimal.valueOf(value).multiply(factor).shortValue());
+  encodedPage.putShort(rowId, (short) (value * factor));
   break;
 case SHORT_INT:
-  encodedPage.putShortInt(rowId,
-  BigDecimal.valueOf(value).multiply(factor).intValue());
+  encodedPage.putShortInt(rowId, (int) (value * factor));
   break;
 case INT:
-  encodedPage.putInt(rowId,
-  BigDecimal.valueOf(value).multiply(factor).intValue());
+  encodedPage.putInt(rowId, (int) (value * factor));
   break;
 case LONG:
-  encodedPage.putLong(rowId,
-  BigDecimal.valueOf(value).multiply(factor).longValue());
+  encodedPage.putLong(rowId, (long) (value * factor));
   break;
 default:
   throw new RuntimeException("internal error: " + debugInfo());
@@ -169,24 +163,19 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
 public void encode(int rowId, double value) {
   switch (targetDataType) {
 

[19/51] [abbrv] carbondata git commit: [CARBONDATA-1417]Added cluster tests for IUD, batch sort and global sort features

[ravipesala]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc39b287/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
--
diff --git 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
new file mode 100644
index 000..d6fa3ca
--- /dev/null
+++ 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
@@ -0,0 +1,3773 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util._
+import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, BeforeAndAfterEach}
+
+/**
+ * Test Class for DataLoadingIUDTestCase to verify all scenerios
+ */
+
+class DataLoadingIUDTestCase extends QueryTest with BeforeAndAfterAll with 
BeforeAndAfter with BeforeAndAfterEach {
+
+  override def beforeAll {
+sql("use default").collect
+sql("drop table if exists t_carbn02").collect
+sql("drop table if exists t_carbn01").collect
+sql("drop table if exists T_Parq1").collect
+sql("drop table if exists table_C21").collect
+sql("drop table if exists t_hive01").collect
+sql("drop table if exists t_carbn2").collect
+sql("drop table if exists t_carbn1").collect
+sql("drop table if exists t1").collect
+sql("drop table if exists t2").collect
+sql("drop table if exists t_carbn21").collect
+sql("drop table if exists t_carbn22").collect
+sql("drop table if exists t_carbn23").collect
+sql("drop table if exists t_carbn24").collect
+sql("drop table if exists t_carbn25").collect
+sql("drop table if exists t_carbn26").collect
+sql("drop table if exists t_carbn27").collect
+sql("drop table if exists t_carbn28").collect
+sql("drop table if exists t_carbn20").collect
+sql("drop table if exists t_carbn30").collect
+sql("drop table if exists t_carbn31").collect
+sql("drop table if exists uniqdata0001_Test").collect
+sql("drop table if exists uniqdata").collect
+sql("drop table if exists uniqdata1").collect
+sql("drop table if exists uniqdata2").collect
+sql("drop table if exists uniqdata023456").collect
+sql("drop table if exists t_carbn01b").collect
+sql("drop table if exists T_Hive1").collect
+sql("drop table if exists T_Hive6").collect
+  }
+
+  override def before(fun: => Any) {
+sql(s"""drop table if exists t_carbn01""").collect
+sql(s"""drop table if exists default.t_carbn01""").collect
+  }
+
+  override def beforeEach(): Unit = {
+sql(s"""drop table if exists t_carbn01""").collect
+sql(s"""drop table if exists default.t_carbn01""").collect
+  }
+
+
+//NA
+test("IUD-01-01-01_001-001", Include) {
+   sql(s"""create table default.t_carbn01b(Active_status String,Item_type_cd 
INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep 
DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name 
String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 
'org.apache.carbondata.format'""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' INTO 
table default.t_carbn01B options ('DELIMITER'=',', 'QUOTECHAR'='\', 
'FILEHEADER'='Active_status,Item_type_cd,Qty_day_avg,Qty_total,Sell_price,Sell_pricep,Discount_price,Profit,Item_code,Item_name,Outlet_name,Update_time,Create_date')""").collect
+  sql("create table T_Hive1(Active_status BOOLEAN, Item_type_cd TINYINT, 
Qty_day_avg SMALLINT, Qty_total INT, Sell_price BIGINT, Sell_pricep FLOAT, 
Discount_price DOUBLE , Profit DECIMAL(3,2), Item_code STRING, Item_name 
VARCHAR(50), Outlet_name CHAR(100), Update_time TIMESTAMP, Create_date DATE) 
row format delimited fields terminated by ',' collection items terminated by 
'$'")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' 

[33/51] [abbrv] carbondata git commit: [CARBONDATA-1465] resolved bug for hive cant query carbon table when column name is in small letters

[ravipesala]

[CARBONDATA-1465] resolved bug for hive cant query carbon table when column 
name is in small letters

1.Resolved bug for hive can't query carbon when column name is in small letters 
 2.Corrected the hive guide there is no need of alter table statement to alter 
location now it is done by CarbonHiveMetastore Event listener itself

This closes #1347


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c15a11d0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c15a11d0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c15a11d0

Branch: refs/heads/branch-1.2
Commit: c15a11d089ed5c9fa679ed7052de481c59188aab
Parents: 887310f
Author: anubhav100 
Authored: Mon Sep 11 15:58:16 2017 +0530
Committer: chenliang613 
Committed: Wed Sep 13 22:46:07 2017 +0800

--
 integration/hive/hive-guide.md   | 1 -
 .../java/org/apache/carbondata/hive/MapredCarbonInputFormat.java | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/c15a11d0/integration/hive/hive-guide.md
--
diff --git a/integration/hive/hive-guide.md b/integration/hive/hive-guide.md
index d554efa..b3848b5 100644
--- a/integration/hive/hive-guide.md
+++ b/integration/hive/hive-guide.md
@@ -91,7 +91,6 @@ $HIVE_HOME/bin/hive
 ### Query data from hive table
 
 ```
-alter table hive_carbon set location '/hive_carbon';
 set hive.mapred.supports.subdirectories=true;
 set mapreduce.input.fileinputformat.input.dir.recursive=true;
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c15a11d0/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
--
diff --git 
a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
 
b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
index 8f3fdce..2e840c0 100644
--- 
a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
+++ 
b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
@@ -163,7 +163,7 @@ public class MapredCarbonInputFormat extends 
CarbonInputFormat
 StringBuilder allColumns = new StringBuilder();
 StringBuilder projectionColumns = new StringBuilder();
 for (CarbonColumn column : carbonColumns) {
-  carbonColumnNames.add(column.getColName());
+  carbonColumnNames.add(column.getColName().toLowerCase());
   allColumns.append(column.getColName() + ",");
 }
 
@@ -172,7 +172,7 @@ public class MapredCarbonInputFormat extends 
CarbonInputFormat
   //verify that the columns parsed by Hive exist in the table
   for (String col : columnNames) {
 //show columns command will return these data
-if (carbonColumnNames.contains(col)) {
+if (carbonColumnNames.contains(col.toLowerCase())) {
   projectionColumns.append(col + ",");
 }
   }

[05/51] [abbrv] carbondata git commit: [CARBONDATA-1442] Refactored Partition-Guide.md

[ravipesala]

[CARBONDATA-1442] Refactored Partition-Guide.md

This closes #1310


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/cd2332e5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/cd2332e5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/cd2332e5

Branch: refs/heads/branch-1.2
Commit: cd2332e5493dfc78683af9c9fb0cfccbe34703ae
Parents: dc7d505
Author: PallaviSingh1992 
Authored: Thu Sep 7 10:32:10 2017 +0530
Committer: Jacky Li 
Committed: Fri Sep 8 22:24:32 2017 +0800

--
 docs/partition-guide.md | 115 ++-
 1 file changed, 71 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/cd2332e5/docs/partition-guide.md
--
diff --git a/docs/partition-guide.md b/docs/partition-guide.md
index 2a0df76..b0b7862 100644
--- a/docs/partition-guide.md
+++ b/docs/partition-guide.md
@@ -17,32 +17,34 @@
 under the License.
 -->
 
-### CarbonData Partition Table Guidance
-This guidance illustrates how to create & use partition table in CarbonData.
+# CarbonData Partition Table Guide
+This tutorial is designed to provide a quick introduction to create and use 
partition table in Apache CarbonData.
 
 * [Create Partition Table](#create-partition-table)
   - [Create Hash Partition Table](#create-hash-partition-table)
   - [Create Range Partition Table](#create-range-partition-table)
   - [Create List Partition Table](#create-list-partition-table)
 * [Show Partitions](#show-partitions)
-* [Maintain the Partitions](#maintain-the-partitions)
+* [Maintaining the Partitions](#maintaining-the-partitions)
 * [Partition Id](#partition-id)
-* [Tips](#tips)
+* [Useful Tips](#useful-tips)
 
-### Create Partition Table
+## Create Partition Table
+
+### Create Hash Partition Table
 
-# Create Hash Partition Table
 ```
CREATE TABLE [IF NOT EXISTS] [db_name.]table_name
 [(col_name data_type , ...)]
PARTITIONED BY (partition_col_name data_type)
STORED BY 'carbondata'
-   [TBLPROPERTIES ('PARTITION_TYPE'='HASH', 
-   'PARTITION_NUM'='N' ...)]  
+   [TBLPROPERTIES ('PARTITION_TYPE'='HASH',
+   'PARTITION_NUM'='N' ...)]
//N is the number of hash partitions
 ```
 
 Example:
+
 ```
create table if not exists hash_partition_table(
   col_A String,
@@ -55,20 +57,25 @@ Example:
tblproperties('partition_type'='Hash','partition_num'='9')
 ```
 
-# Create Range Partition Table
+### Create Range Partition Table
+
 ```
CREATE TABLE [IF NOT EXISTS] [db_name.]table_name
 [(col_name data_type , ...)]
PARTITIONED BY (partition_col_name data_type)
STORED BY 'carbondata'
-   [TBLPROPERTIES ('PARTITION_TYPE'='RANGE', 
+   [TBLPROPERTIES ('PARTITION_TYPE'='RANGE',
'RANGE_INFO'='2014-01-01, 2015-01-01, 2016-01-01' ...)]
 ```
-Notes: 
-1. The 'RANGE_INFO' defined in table properties must be in ascending order.
-2. If the partition column is Date/Timestamp type, the format could be defined 
in CarbonProperties. By default it's -MM-dd.
+
+**Note:**
+
+- The 'RANGE_INFO' must be defined in ascending order in the table properties.
+
+- The default format for partition column of Date/Timestamp type is 
-MM-dd. Alternate formats for Date/Timestamp could be defined in 
CarbonProperties.
 
 Example:
+
 ```
create table if not exists hash_partition_table(
   col_A String,
@@ -82,19 +89,21 @@ Example:
'range_info'='2015-01-01, 2016-01-01, 2017-01-01, 2017-02-01')
 ```
 
-# Create List Partition Table
+### Create List Partition Table
+
 ```
CREATE TABLE [IF NOT EXISTS] [db_name.]table_name
 [(col_name data_type , ...)]
PARTITIONED BY (partition_col_name data_type)
STORED BY 'carbondata'
-   [TBLPROPERTIES ('PARTITION_TYPE'='LIST', 
+   [TBLPROPERTIES ('PARTITION_TYPE'='LIST',
'LIST_INFO'='A, B, C' ...)]
 ```
-Notes:
-1. List partition support list info in one level group. 
+**Note :**
+- List partition supports list info in one level group.
+
+Example:
 
-Example：
 ```
create table if not exists hash_partition_table(
   col_B Int,
@@ -109,41 +118,53 @@ Example：
 ```
 
 
-### Show Partitions
-Execute following command to get the partition information
+## Show Partitions
+The following command is executed to get the partition information of the table
+
 ```
SHOW PARTITIONS [db_name.]table_name
-
 ```
 
-### Maintain the Partitions
-# Add a new partition
+## Maintaining the Partitions
+### Add a new partition
+
 ```
ALTER TABLE [db_name].table_name ADD PARTITION('new_partition')
 ```
-# Split a partition
+### Split 

[18/51] [abbrv] carbondata git commit: [CARBONDATA-1417]Added cluster tests for IUD, batch sort and global sort features

[ravipesala]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc39b287/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
--
diff --git 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
new file mode 100644
index 000..bd8a5ff
--- /dev/null
+++ 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
@@ -0,0 +1,621 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+  * Test Class for globalsort1TestCase to verify all scenerios
+  */
+
+class GlobalSortTestCase extends QueryTest with BeforeAndAfterAll {
+
+  override def beforeAll {
+sql(s"""drop table if exists uniqdata11""").collect
+sql(s"""drop table if exists uniqdataquery1""").collect
+  }
+
+
+  //Carbon-Loading-Optimizations-Global-Sort-01-01-01
+  test("Carbon-Loading-Optimizations-Global-Sort-01-01-01", Include) {
+sql(s"""drop table if exists uniqdata11""".stripMargin).collect
+sql(
+  s"""CREATE TABLE uniqdata11 (CUST_ID int,CUST_NAME String,
+ |ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp,
+ |BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 
decimal(30,10),
+ |DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 
double,
+ |INTEGER_COLUMN1 int) STORED BY 
'carbondata'""".stripMargin.replaceAll(System
+.lineSeparator, "")).collect
+
+sql(
+  s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv'
+ | into table uniqdata11 OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"',
+ | 
'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,
+ | 
DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,
+ | 
Double_COLUMN2,INTEGER_COLUMN1')""".stripMargin.replaceAll(System.lineSeparator,
 ""))
+  .collect
+
+sql(s"""select * from uniqdata11""").collect
+sql(s"""drop table if exists uniqdata11""").collect
+
+  }
+
+
+  //Carbon-Loading-Optimizations-Global-Sort-01-01-02
+  test("Carbon-Loading-Optimizations-Global-Sort-01-01-02", Include) {
+sql(
+  s"""CREATE TABLE uniqdata11 (CUST_ID int,CUST_NAME 
String,ACTIVE_EMUI_VERSION string,
+ |DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 
bigint,
+ |DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 
decimal(36,10),Double_COLUMN1 double,
+ |Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 
'carbondata'""".stripMargin
+.replaceAll(System.lineSeparator, "")).collect
+
+sql(
+  s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' 
into table
+ | uniqdata11 OPTIONS('DELIMITER'=',' , 
'QUOTECHAR'='"','COMMENTCHAR'='#',
+ | 'MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='FORCE',
+ | 
'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,
+ | 
BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,
+ | INTEGER_COLUMN1')""".stripMargin.replaceAll(System.lineSeparator, 
"")).collect
+
+
+sql(s"""select * from uniqdata11""").collect
+sql(s"""drop table if exists uniqdata11""").collect
+
+  }
+
+
+  //Carbon-Loading-Optimizations-Global-Sort-01-01-03
+  test("Carbon-Loading-Optimizations-Global-Sort-01-01-03", Include) {
+sql(s"""CREATE TABLE uniqdata11 (CUST_ID int,CUST_NAME 
String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 
bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 
decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 
int) STORED BY 'carbondata'""").collect
+
+sql(s"""LOAD DATA INPATH '$resourcesPath/Data/batchsort/folder1/folder2' 

[28/51] [abbrv] carbondata git commit: [CARBONDATA-1400] Fix bug of array column out of bound when writing carbondata file

[ravipesala]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8c1ddbf2/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala
new file mode 100644
index 000..f4fd168
--- /dev/null
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.integration.spark.testsuite.complexType
+
+import java.io.{File, FileOutputStream, PrintStream}
+
+import scala.collection.mutable
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class TestComplexTypeWithBigArray extends QueryTest with BeforeAndAfterAll {
+
+  val filePath = "./list.csv"
+  val file = new File(filePath)
+
+  override def beforeAll: Unit = {
+// write a CSV containing 32000 row, each row has an array with 10 elements
+val out = new PrintStream(new FileOutputStream(file))
+(1 to 33000).foreach(i=>out.println(s"$i,$i$$1"))
+out.close()
+  }
+
+  test("test with big string array") {
+sql("DROP TABLE IF EXISTS big_array")
+sql(
+  """
+| CREATE TABLE big_array(
+|  value BIGINT,
+|  list ARRAY
+|  )
+| STORED BY 'carbondata'
+  """.stripMargin)
+sql(
+  s"""
+ | LOAD DATA LOCAL INPATH '${file.getAbsolutePath}'
+ | INTO TABLE big_array
+ | OPTIONS ('header'='false')
+  """.stripMargin)
+checkAnswer(
+  sql("select count(*) from big_array"),
+  Row(33000)
+)
+checkAnswer(
+  sql("select * from big_array limit 1"),
+  Row(1, mutable.WrappedArray.make[String](Array("1", "1")))
+)
+checkAnswer(
+  sql("select list[1] from big_array limit 1"),
+  Row("1")
+)
+checkAnswer(
+  sql("select count(*) from big_array where list[0] = '1'"),
+  Row(1)
+)
+checkAnswer(
+  sql("select count(*) from big_array where array_contains(list, '1') "),
+  Row(33000)
+)
+if (sqlContext.sparkContext.version.startsWith("2.")) {
+  // explode UDF is supported start from spark 2.0
+  checkAnswer(
+sql("select count(x) from (select explode(list) as x from big_array)"),
+Row(66000)
+  )
+}
+checkAnswer(
+  sql("select * from big_array where value = 15000"),
+  Row(15000, mutable.WrappedArray.make[String](Array("15000", "1")))
+)
+checkAnswer(
+  sql("select * from big_array where value = 32500"),
+  Row(32500, mutable.WrappedArray.make[String](Array("32500", "1")))
+)
+checkAnswer(
+  sql("select count(list) from big_array"),
+  Row(33000)
+)
+sql("DROP TABLE big_array")
+  }
+
+  test("test with big int array") {
+sql("DROP TABLE IF EXISTS big_array")
+sql(
+  """
+| CREATE TABLE big_array(
+|  value BIGINT,
+|  list ARRAY
+|  )
+| STORED BY 'carbondata'
+  """.stripMargin)
+sql(
+  s"""
+ | LOAD DATA LOCAL INPATH '${file.getAbsolutePath}'
+ | INTO TABLE big_array
+ | OPTIONS ('header'='false')
+  """.stripMargin)
+checkAnswer(
+  sql("select count(*) from big_array"),
+  Row(33000)
+)
+checkAnswer(
+  sql("select * from big_array limit 1"),
+  Row(1, mutable.WrappedArray.make[String](Array(1, 1)))
+)
+checkAnswer(
+  sql("select list[1] from big_array limit 1"),
+  Row(1)
+)
+checkAnswer(
+  sql("select count(*) from big_array where list[0] = 1"),
+  Row(1)
+)
+checkAnswer(
+  sql("select count(*) from big_array where array_contains(list, 1) "),
+  Row(33000)
+)
+if 

Jenkins build became unstable: carbondata-master-spark-1.6 » Apache CarbonData :: Spark Common Test #1879

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-1.6 #1879

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-1.6 #1880

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-1.6 » Apache CarbonData :: Spark Common Test #1880

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-1.6 » Apache CarbonData :: Spark Common Test #1882

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-1.6 #1882

[Apache Jenkins Server]

See 

[1/2] carbondata git commit: [CARBONDATA-1316] Support drop partition function

[qiangcai]

Repository: carbondata
Updated Branches:
  refs/heads/master fe36e3bc9 -> cb51b8621


http://git-wip-us.apache.org/repos/asf/carbondata/blob/cb51b862/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index 0c59bd9..3646fad 100644
--- 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -260,8 +260,8 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 checkAnswer(result_after, result_origin)
 
 val result_after1 = sql(s"select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area where area < 'OutSpace' ")
-val rssult_origin1 = sql(s"select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area_origin where area < 'OutSpace' ")
-checkAnswer(result_after1, rssult_origin1)
+val result_origin1 = sql(s"select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area_origin where area < 'OutSpace' ")
+checkAnswer(result_after1, result_origin1)
 
 val result_after2 = sql("select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area where area <= 'OutSpace' ")
 val result_origin2 = sql("select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area_origin where area <= 'OutSpace' ")
@@ -279,28 +279,24 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 val result_origin5 = sql("select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area_origin where area >= 'OutSpace' ")
 checkAnswer(result_after5, result_origin5)
 
-sql("""ALTER TABLE list_table_area ADD PARTITION ('One', '(Two, Three)', 
'Four')""".stripMargin)
-val carbonTable1 = 
CarbonMetadata.getInstance().getCarbonTable("default_list_table_area")
-val partitionInfo1 = 
carbonTable1.getPartitionInfo(carbonTable.getFactTableName)
-val partitionIds1 = partitionInfo1.getPartitionIds
-val new_list_info = partitionInfo1.getListInfo
-assert(partitionIds1 == List(0, 1, 2, 3, 4, 5, 6, 7, 
8).map(Integer.valueOf(_)).asJava)
-assert(partitionInfo1.getMAX_PARTITION == 8)
-assert(partitionInfo1.getNumPartitions == 9)
-assert(new_list_info.get(0).get(0) == "Asia")
-assert(new_list_info.get(1).get(0) == "America")
-assert(new_list_info.get(2).get(0) == "Europe")
-assert(new_list_info.get(3).get(0) == "OutSpace")
-assert(new_list_info.get(4).get(0) == "Hi")
-assert(new_list_info.get(5).get(0) == "One")
-assert(new_list_info.get(6).get(0) == "Two")
-assert(new_list_info.get(6).get(1) == "Three")
-assert(new_list_info.get(7).get(0) == "Four")
-validateDataFiles("default_list_table_area", "0", Seq(0, 1, 2, 4))
-
-val result_after6 = sql("select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area")
-val result_origin6 = sql("""select id, vin, logdate, phonenumber, country, 
area, salary from list_table_area_origin""")
-checkAnswer(result_after6, result_origin6)
+intercept[Exception]  { sql("""ALTER TABLE DROP PARTITION(0)""")}
+intercept[Exception]  { sql("""ALTER TABLE DROP PARTITION(0) WITH DATA""")}
+
+sql("""ALTER TABLE list_table_area DROP PARTITION(2) WITH DATA""")
+val carbonTable2 = 
CarbonMetadata.getInstance().getCarbonTable("default_list_table_area")
+val partitionInfo2 = 
carbonTable2.getPartitionInfo(carbonTable.getFactTableName)
+val partitionIds2 = partitionInfo2.getPartitionIds
+val list_info2 = partitionInfo2.getListInfo
+assert(partitionIds2 == List(0, 1, 3, 4, 5).map(Integer.valueOf(_)).asJava)
+assert(partitionInfo2.getMAX_PARTITION == 5)
+assert(partitionInfo2.getNumPartitions == 5)
+assert(list_info2.get(0).get(0) == "Asia")
+assert(list_info2.get(1).get(0) == "Europe")
+assert(list_info2.get(2).get(0) == "OutSpace")
+assert(list_info2.get(3).get(0) == "Hi")
+validateDataFiles("default_list_table_area", "0", Seq(0, 1, 4))
+checkAnswer(sql("select id, vin, logdate, phonenumber, country, area, 
salary from list_table_area"),
+  sql("select id, vin, logdate, phonenumber, country, area, salary from 
list_table_area_origin where area <> 'America' "))
   }
 
   test("Alter table add partition: Range Partition") {
@@ -309,9 +305,9 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 val partitionInfo = 
carbonTable.getPartitionInfo(carbonTable.getFactTableName)
 val partitionIds = 

[2/2] carbondata git commit: [CARBONDATA-1316] Support drop partition function

[qiangcai]

[CARBONDATA-1316] Support drop partition function

This closes #1317


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/cb51b862
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/cb51b862
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/cb51b862

Branch: refs/heads/master
Commit: cb51b86218cd815167f7c702b643ed0852c7f3dc
Parents: fe36e3b
Author: lionelcao 
Authored: Mon Sep 4 15:38:44 2017 +0800
Committer: QiangCai 
Committed: Mon Sep 18 17:19:22 2017 +0800

--
 .../core/metadata/schema/PartitionInfo.java |   5 +
 .../hadoop/api/CarbonTableInputFormat.java  |  23 ++-
 .../spark/partition/DropPartitionCallable.java  |  39 +
 .../org/apache/carbondata/spark/KeyVal.scala|   4 +-
 .../spark/rdd/AlterTableLoadPartitionRDD.scala  | 141 +++
 .../spark/rdd/AlterTableSplitPartitionRDD.scala | 146 
 .../spark/rdd/CarbonScanPartitionRDD.scala  |  29 ++--
 .../apache/carbondata/spark/rdd/Compactor.scala |   3 +-
 .../spark/rdd/DataManagementFunc.scala  |  50 +++---
 .../carbondata/spark/rdd/PartitionDropper.scala | 122 +
 .../spark/rdd/PartitionSplitter.scala   |  36 ++--
 .../carbondata/spark/util/CommonUtil.scala  |   2 +-
 .../spark/util/GlobalDictionaryUtil.scala   |   3 +-
 .../command/carbonTableSchemaCommon.scala   |  25 ++-
 .../org/apache/spark/util/PartitionUtils.scala  |  15 +-
 .../spark/rdd/CarbonDataRDDFactory.scala|   8 +-
 .../execution/command/carbonTableSchema.scala   |   2 +-
 .../spark/rdd/CarbonDataRDDFactory.scala| 103 ---
 .../execution/command/carbonTableSchema.scala   | 145 +++-
 .../sql/parser/CarbonSpark2SqlParser.scala  |  16 +-
 .../partition/TestAlterPartitionTable.scala | 171 +++
 .../processing/merger/CarbonDataMergerUtil.java |   5 +-
 .../processing/spliter/RowResultProcessor.java  | 105 
 .../spliter/RowResultSpliterProcessor.java  | 105 
 .../exception/AlterPartitionSliceException.java |  78 +
 .../exception/SliceSpliterException.java|  78 -
 26 files changed, 978 insertions(+), 481 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/cb51b862/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
index 4b0bc3e..d0c4447 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
@@ -92,6 +92,11 @@ public class PartitionInfo implements Serializable {
 numPartitions = numPartitions - 1 + newPartitionNumbers;
   }
 
+  public void dropPartition(int index) {
+partitionIds.remove(index);
+numPartitions--;
+  }
+
   public List getColumnSchemaList() {
 return columnSchemaList;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/cb51b862/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index dcc75bd..9076233 100644
--- 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -306,7 +306,7 @@ public class CarbonTableInputFormat extends 
FileInputFormat {
 // prune partitions for filter query on partition table
 BitSet matchedPartitions = null;
 if (partitionInfo != null) {
-  matchedPartitions = setMatchedPartitions(null, filter, partitionInfo);
+  matchedPartitions = setMatchedPartitions(null, filter, partitionInfo, 
null);
   if (matchedPartitions != null) {
 if (matchedPartitions.cardinality() == 0) {
   return new ArrayList();
@@ -366,9 +366,11 @@ public class CarbonTableInputFormat extends 
FileInputFormat {
   TableProvider tableProvider = new SingleTableProvider(carbonTable);
   // prune partitions for filter query on partition table
   String partitionIds = job.getConfiguration().get(ALTER_PARTITION_ID);
+  // matchedPartitions records partitionIndex, not partitionId
   BitSet matchedPartitions = null;
   if (partitionInfo != null) {
-matchedPartitions = setMatchedPartitions(partitionIds, 

carbondata git commit: [CARBONDATA-1478] Update compaction documentation

[jackylk]

Repository: carbondata
Updated Branches:
  refs/heads/master c2574688f -> fe36e3bc9


[CARBONDATA-1478] Update compaction documentation

In order to easier to understand, update the description about compaction.

This closes #1357


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/fe36e3bc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/fe36e3bc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/fe36e3bc

Branch: refs/heads/master
Commit: fe36e3bc979958cf94bcefaeaa651903017ce084
Parents: c257468
Author: xubo245 <601450...@qq.com>
Authored: Wed Sep 13 20:24:40 2017 +0800
Committer: Jacky Li 
Committed: Mon Sep 18 17:10:59 2017 +0800

--
 docs/data-management.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/fe36e3bc/docs/data-management.md
--
diff --git a/docs/data-management.md b/docs/data-management.md
index 81866a1..b1a3eef 100644
--- a/docs/data-management.md
+++ b/docs/data-management.md
@@ -106,7 +106,7 @@ Example :
 CLEAN FILES FOR TABLE table1
 ```
 
- This DML will physically delete the segment which are "Marked for delete" 
immediately.
+ This DML will physically delete the segment which are "Marked for delete" and 
"Compacted" immediately.
 
 ## Compacting Data
   
@@ -135,7 +135,7 @@ CLEAN FILES FOR TABLE table1

 | Parameter | Default | Application | Description | Valid Values |
 
|-|-|-||--|
-| carbon.compaction.level.threshold | 4, 3 | Minor | This property is for 
minor compaction which decides how many segments to be merged. Example: If it 
is set as 2, 3 then minor compaction will be triggered for every 2 segments. 3 
is the number of level 1 compacted segment which is further compacted to new 
segment. | NA |
+| carbon.compaction.level.threshold | 4, 3 | Minor | This property is for 
minor compaction which decides how many segments to be merged. Example: If it 
is set as "2, 3", then minor compaction will be triggered for every 2 segments 
in level 1. 3 is the number of level 1 compacted segment which is further 
compacted to new segment in level 2. | NA |
 | carbon.major.compaction.size | 1024 MB | Major | Major compaction size can 
be configured using this parameter. Sum of the segments which is below this 
threshold will be merged. | NA |
 | carbon.numberof.preserve.segments | 0 | Minor/Major | This property 
configures number of segments to preserve from being compacted. Example: 
carbon.numberof.preserve.segments=2 then 2 latest segments will always be 
excluded from the compaction. No segments will be preserved by default. | 0-100 
|
 | carbon.allowed.compaction.days | 0 | Minor/Major | Compaction will merge the 
segments which are loaded within the specific number of days configured. 
Example: If the configuration is 2, then the segments which are loaded in the 
time frame of 2 days only will get merged. Segments which are loaded 2 days 
apart will not be merged. This is disabled by default. | 0-100 |

[04/51] [abbrv] carbondata git commit: Fix examples metastore issues

[ravipesala]

Fix examples metastore issues

This closes #1339


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dc7d5057
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dc7d5057
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dc7d5057

Branch: refs/heads/branch-1.2
Commit: dc7d50574a9aefd32139703e0f18b5863a886b26
Parents: 01492fc
Author: chenliang613 
Authored: Thu Sep 7 23:22:50 2017 +0800
Committer: chenliang613 
Committed: Thu Sep 7 23:50:40 2017 +0800

--
 .../org/apache/carbondata/examples/CarbonDataFrameExample.scala| 2 +-
 .../org/apache/carbondata/examples/CarbonPartitionExample.scala| 2 +-
 .../org/apache/carbondata/examples/CarbonSessionExample.scala  | 2 +-
 .../org/apache/carbondata/examples/CarbonSortColumnsExample.scala  | 2 +-
 .../org/apache/carbondata/examples/DataUpdateDeleteExample.scala   | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc7d5057/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
--
diff --git 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
index 74740e7..ac198d8 100644
--- 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
+++ 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
@@ -42,7 +42,7 @@ object CarbonDataFrameExample {
   .master("local")
   .appName("CarbonDataFrameExample")
   .config("spark.sql.warehouse.dir", warehouse)
-  .getOrCreateCarbonSession(storeLocation, metastoredb)
+  .getOrCreateCarbonSession(storeLocation)
 
 spark.sparkContext.setLogLevel("ERROR")
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc7d5057/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
--
diff --git 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
index ca0501c..d8aca6b 100644
--- 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
+++ 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
@@ -47,7 +47,7 @@ object CarbonPartitionExample {
   .master("local")
   .appName("CarbonPartitionExample")
   .config("spark.sql.warehouse.dir", warehouse)
-  .getOrCreateCarbonSession(storeLocation, metastoredb)
+  .getOrCreateCarbonSession(storeLocation)
 
 spark.sparkContext.setLogLevel("WARN")
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc7d5057/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
--
diff --git 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index 7432fe9..3b2094a 100644
--- 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -45,7 +45,7 @@ object CarbonSessionExample {
   .appName("CarbonSessionExample")
   .config("spark.sql.warehouse.dir", warehouse)
   .config("spark.driver.host", "localhost")
-  .getOrCreateCarbonSession(storeLocation, metastoredb)
+  .getOrCreateCarbonSession(storeLocation)
 
 spark.sparkContext.setLogLevel("WARN")
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc7d5057/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSortColumnsExample.scala
--
diff --git 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSortColumnsExample.scala
 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSortColumnsExample.scala
index 7baee56..2f6322c 100644
--- 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSortColumnsExample.scala
+++ 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSortColumnsExample.scala
@@ -44,7 +44,7 @@ object CarbonSortColumnsExample {
   .appName("CarbonSortColumnsExample")
   .config("spark.sql.warehouse.dir", warehouse)
   .config("spark.driver.host", "localhost")
-  

[07/51] [abbrv] carbondata git commit: [CARBONDATA-1456]Regenerate cached hive results if cluster testcases fail

[ravipesala]

[CARBONDATA-1456]Regenerate cached hive results if cluster testcases fail

This closes #1332


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0ebdc94c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0ebdc94c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0ebdc94c

Branch: refs/heads/branch-1.2
Commit: 0ebdc94cf630b54605f38443572217bbde493ae5
Parents: 2d75c46
Author: sraghunandan 
Authored: Wed Sep 6 15:09:44 2017 +0530
Committer: Ravindra Pesala 
Committed: Sat Sep 9 11:03:46 2017 +0530

--
 integration/spark-common-cluster-test/pom.xml   |  1 -
 .../spark/sql/common/util/QueryTest.scala   | 32 +---
 2 files changed, 21 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ebdc94c/integration/spark-common-cluster-test/pom.xml
--
diff --git a/integration/spark-common-cluster-test/pom.xml 
b/integration/spark-common-cluster-test/pom.xml
index 935e55b..0f71653 100644
--- a/integration/spark-common-cluster-test/pom.xml
+++ b/integration/spark-common-cluster-test/pom.xml
@@ -143,7 +143,6 @@
   
   
 true
-GMT
 ${spark.master.url}
 ${hdfs.url}
   

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ebdc94c/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
--
diff --git 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
index 7c78b10..0c04b5e 100644
--- 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
+++ 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
@@ -40,8 +40,6 @@ class QueryTest extends PlanTest with Suite {
 
   val DOLLAR = "$"
 
-  // Timezone is fixed to America/Los_Angeles for those timezone sensitive 
tests (timestamp_*)
-  TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
   // Add Locale setting
   Locale.setDefault(Locale.US)
 
@@ -84,22 +82,34 @@ class QueryTest extends PlanTest with Suite {
 checkAnswer(df, expectedAnswer.collect())
   }
 
-  protected def checkAnswer(carbon: String, hive: String, 
uniqueIdentifier:String): Unit = {
-val path = TestQueryExecutor.hiveresultpath + "/"+uniqueIdentifier
+  protected def checkAnswer(carbon: String, hive: String, uniqueIdentifier: 
String): Unit = {
+val path = TestQueryExecutor.hiveresultpath + "/" + uniqueIdentifier
 if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
-  val objinp = new ObjectInputStream(FileFactory.getDataInputStream(path, 
FileFactory.getFileType(path)))
+  val objinp = new ObjectInputStream(FileFactory
+.getDataInputStream(path, FileFactory.getFileType(path)))
   val rows = objinp.readObject().asInstanceOf[Array[Row]]
   objinp.close()
-  checkAnswer(sql(carbon), rows)
+  QueryTest.checkAnswer(sql(carbon), rows) match {
+case Some(errorMessage) => {
+  FileFactory.deleteFile(path, FileFactory.getFileType(path))
+  writeAndCheckAnswer(carbon, hive, path)
+}
+case None =>
+  }
 } else {
-  val rows = sql(hive).collect()
-  val obj = new ObjectOutputStream(FileFactory.getDataOutputStream(path, 
FileFactory.getFileType(path)))
-  obj.writeObject(rows)
-  obj.close()
-  checkAnswer(sql(carbon), rows)
+  writeAndCheckAnswer(carbon, hive, path)
 }
   }
 
+  private def writeAndCheckAnswer(carbon: String, hive: String, path: String): 
Unit = {
+val rows = sql(hive).collect()
+val obj = new ObjectOutputStream(FileFactory.getDataOutputStream(path, 
FileFactory
+  .getFileType(path)))
+obj.writeObject(rows)
+obj.close()
+checkAnswer(sql(carbon), rows)
+  }
+
   protected def checkAnswer(carbon: String, expectedAnswer: Seq[Row], 
uniqueIdentifier:String): Unit = {
 checkAnswer(sql(carbon), expectedAnswer)
   }

[24/51] [abbrv] carbondata git commit: [CARBONDATA-1423] added integration test cases for presto

[ravipesala]

[CARBONDATA-1423] added integration test cases for presto

This closes #1303


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b6727d75
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b6727d75
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b6727d75

Branch: refs/heads/branch-1.2
Commit: b6727d75d2a79498c6861959bba24d96fd075108
Parents: 9f0ac24
Author: anubhav100 
Authored: Tue Aug 29 14:19:31 2017 +0530
Committer: chenliang613 
Committed: Wed Sep 13 12:11:07 2017 +0800

--
 integration/presto/pom.xml  | 319 +++
 .../presto/src/test/resources/alldatatype.csv   |  11 +
 .../presto/src/test/resources/log4j.properties  |  11 +
 .../integrationtest/PrestoAllDataTypeTest.scala | 403 +
 .../carbondata/presto/server/PrestoServer.scala | 170 ++
 .../presto/util/CarbonDataStoreCreator.scala| 559 +++
 6 files changed, 1373 insertions(+), 100 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6727d75/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 617ce93..924a2be 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -15,9 +15,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 -->
-http://maven.apache.org/POM/4.0.0;
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 
   4.0.0
 
@@ -38,62 +36,35 @@
   
 
   
+
+
+  com.fasterxml.jackson.core
+  jackson-databind
+  2.8.1
+  
+
+  com.fasterxml.jackson.core
+  jackson-annotations
+
+  
+
+
+
+  com.fasterxml.jackson.core
+  jackson-annotations
+  2.8.1
+  provided
+
+
+
 
   org.apache.carbondata
   carbondata-hadoop
   ${project.version}
   
 
-  org.apache.spark
-  spark-network-shuffle_2.11
-
-
-  org.apache.spark
-  spark-sketch_2.11
-
-
-  org.slf4j
-  slf4j-log4j12
-
-
-  net.java.dev.jets3t
-  jets3t
-
-
-  javax.servlet
-  javax.servlet-api
-
-
-  org.apache.commons
-  commons-math3
-
-
-  org.apache.httpcomponents
-  httpclient
-
-
-  org.antlr
-  antlr4-runtime
-
-
-  com.esotericsoftware
-  minlog
-
-
-  org.codehaus.janino
-  janino
-
-
-  net.jpountz.lz4
-  lz4
-
-
-  net.sf.py4j
-  py4j
-
-
-  org.spark-project.spark
-  unused
+  com.sun.jersey
+  jersey-core
 
 
   org.apache.hadoop
@@ -172,6 +143,7 @@
   io.dropwizard.metrics
   metrics-graphite
 
+
 
   com.google.code.findbugs
   jsr305
@@ -254,7 +226,30 @@
 
   
 
-
+
+  com.facebook.presto
+  presto-tests
+  test
+  ${presto.version}
+  
+
+  org.openjdk.jol
+  jol-core
+
+
+  org.antlr
+  antlr4-runtime
+
+
+  com.fasterxml.jackson.core
+  jackson-databind
+
+
+  org.apache.spark
+  spark-sql_2.10
+
+  
+
 
   io.airlift
   bootstrap
@@ -305,26 +300,82 @@
   json
   0.144
   
+  
+
+  org.slf4j
+  slf4j-jdk14
+
+
+  org.slf4j
+  slf4j-log4j12
+
+
+  org.slf4j
+  log4j-over-slf4j
+
+
+  com.fasterxml.jackson.core
+  jackson-databind
+
+
+  com.fasterxml.jackson.core
+  jackson-annotations
+
+
+  com.fasterxml.jackson.core
+  jackson-core
+
+
+  
 
 
+  org.scalatest
+  scalatest_${scala.binary.version}
+  2.2.1
+  test
+
+
+
   io.airlift
   units
   1.0
   provided
+  
+
+  org.slf4j
+  slf4j-jdk14
+
+
+  org.slf4j
+  

[03/51] [abbrv] carbondata git commit: [CARBONDATA-1461]resolved bug for date and timestamp in hive

[ravipesala]

[CARBONDATA-1461]resolved bug for date and timestamp in hive

This closes #1338


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/01492fce
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/01492fce
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/01492fce

Branch: refs/heads/branch-1.2
Commit: 01492fce4713a4be47397e66d2b1c05dae102df7
Parents: 531dcd2
Author: anubhav100 
Authored: Thu Sep 7 17:37:43 2017 +0530
Committer: chenliang613 
Committed: Thu Sep 7 21:17:16 2017 +0800

--
 .../carbondata/hive/CarbonDictionaryDecodeReadSupport.java  | 9 +++--
 1 file changed, 3 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/01492fce/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
--
diff --git 
a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
 
b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
index 5eae253..b0adc69 100644
--- 
a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
+++ 
b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
@@ -34,7 +34,6 @@ import 
org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.core.util.path.CarbonStorePath;
-
 import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
 
 import org.apache.hadoop.hive.common.type.HiveDecimal;
@@ -43,18 +42,16 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
 import org.apache.hadoop.hive.serde2.io.TimestampWritable;
-
 import org.apache.hadoop.io.ArrayWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
-
 import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
 import org.apache.spark.sql.catalyst.util.GenericArrayData;
 
 /**
- *  This is the class to decode dictionary encoded column data back to its 
original value.
+ * This is the class to decode dictionary encoded column data back to its 
original value.
  */
 public class CarbonDictionaryDecodeReadSupport implements 
CarbonReadSupport {
 
@@ -233,9 +230,9 @@ public class CarbonDictionaryDecodeReadSupport 
implements CarbonReadSupport

[41/51] [abbrv] carbondata git commit: [CARBONDATA-1482] fixed the failing test cases of presto integration

[ravipesala]

[CARBONDATA-1482] fixed the failing test cases of presto integration

1.builds are passing
2.added dependency for spark guava
3.throw exception in case carbondatastore creator is unable to load the data
4.fixed the timestamp issue due to which test cases are failing

This closes #1360


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d60d973d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d60d973d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d60d973d

Branch: refs/heads/branch-1.2
Commit: d60d973dfbb69c1b740a6a00cab878bc168c14ae
Parents: 6f20437
Author: anubhav100 
Authored: Fri Sep 15 15:12:10 2017 +0530
Committer: Ravindra Pesala 
Committed: Fri Sep 15 19:25:10 2017 +0530

--
 integration/presto/pom.xml  |  7 
 .../integrationtest/PrestoAllDataTypeTest.scala | 26 +++---
 .../presto/util/CarbonDataStoreCreator.scala| 36 
 3 files changed, 42 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/d60d973d/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index b23b1be..50e6349 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -438,6 +438,13 @@
 
   
 
+
+
+  org.apache.spark
+  spark-network-common_2.11
+  test
+  2.1.0
+
   
 
   

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d60d973d/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
--
diff --git 
a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
 
b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
index 1743be6..433ddd4 100644
--- 
a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
+++ 
b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
@@ -14,12 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.carbondata.presto.integrationtest
 
 import java.io.File
 
 import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
-import util.CarbonDataStoreCreator
 
 import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.presto.server.PrestoServer
@@ -35,8 +35,10 @@ class PrestoAllDataTypeTest extends FunSuiteLike with 
BeforeAndAfterAll {
   private val storePath = s"$rootPath/integration/presto/target/store"
 
   override def beforeAll: Unit = {
+import org.apache.carbondata.presto.util.CarbonDataStoreCreator
 CarbonDataStoreCreator
-  .createCarbonStore(storePath, 
s"$rootPath/integration/presto/src/test/resources/alldatatype.csv")
+  .createCarbonStore(storePath,
+s"$rootPath/integration/presto/src/test/resources/alldatatype.csv")
 logger.info(s"\nCarbon store is created at location: $storePath")
 PrestoServer.startServer(storePath)
   }
@@ -91,21 +93,21 @@ class PrestoAllDataTypeTest extends FunSuiteLike with 
BeforeAndAfterAll {
   test("test the result for count()clause with distinct operator on decimal 
column in presto") {
 val actualResult: List[Map[String, Any]] = PrestoServer
   .executeQuery("SELECT COUNT(DISTINCT BONUS) AS RESULT FROM 
TESTDB.TESTTABLE ")
-val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 7))
-actualResult.equals(expectedResult)
+val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 10))
+assert(actualResult.equals(expectedResult))
   }
   test("test the result for count()clause with out  distinct operator on 
decimal column in presto")
   {
 val actualResult: List[Map[String, Any]] = PrestoServer
   .executeQuery("SELECT COUNT(BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
 val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 10))
-actualResult.equals(expectedResult)
+assert(actualResult.equals(expectedResult))
   }
   test("test the result for sum()with out distinct operator for decimal column 
in presto") {
 val actualResult: List[Map[String, Any]] = PrestoServer
   .executeQuery("SELECT SUM(DISTINCT BONUS) AS RESULT FROM 
TESTDB.TESTTABLE ")
-val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 54))
-actualResult.equals(expectedResult)
+val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 
20774.6475))
+

[42/51] [abbrv] carbondata git commit: [CARBONDATA-1380] Fixed updation of load fail to table status file to make overwrite queries work properly

[ravipesala]

[CARBONDATA-1380] Fixed updation of load fail to table status file to make 
overwrite queries work properly

Tablestatus file is not updated when load fails. It should be updated with fail 
status otherwise overwrite queries does not work properly.

This closes #1256


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/de445bb6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/de445bb6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/de445bb6

Branch: refs/heads/branch-1.2
Commit: de445bb66c48b9d2db0cf87d03c0af171489644b
Parents: d60d973
Author: Ravindra Pesala 
Authored: Mon Aug 14 18:58:25 2017 +0530
Committer: Ravindra Pesala 
Committed: Sat Sep 16 22:21:36 2017 +0530

--
 .../InsertIntoCarbonTableTestCase.scala | 20 +++-
 .../spark/rdd/CarbonDataRDDFactory.scala| 47 +++
 .../spark/rdd/CarbonDataRDDFactory.scala| 48 
 3 files changed, 78 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/de445bb6/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
index d1bf28b..8a084ae 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -54,7 +54,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with 
BeforeAndAfterAll {
  checkAnswer(
  sql("select 
imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper
 
atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription
 from TCarbonSource order by 
imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNum
 
ber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription"),
  sql("select 

[30/51] [abbrv] carbondata git commit: [CARBONDATA-1400] Fix bug of array column out of bound when writing carbondata file

[ravipesala]

[CARBONDATA-1400] Fix bug of array column out of bound when writing carbondata 
file

If there is a big array in input csv file, when loading carbondata table, it 
may throw ArrayIndexOutOfBoundException because data exceed page size (32000 
rows)

This PR fixed it by changing complex column encoding to 
DirectCompressionEncoding
This PR added a test case to test input data with big array

This closes #1273


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8c1ddbf2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8c1ddbf2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8c1ddbf2

Branch: refs/heads/branch-1.2
Commit: 8c1ddbf2a6ba74a0a6d1333d95d0f6ad70297c01
Parents: b414393
Author: Jacky Li 
Authored: Tue Sep 12 09:33:20 2017 +0800
Committer: Ravindra Pesala 
Committed: Wed Sep 13 17:08:40 2017 +0530

--
 .../cache/dictionary/ColumnDictionaryInfo.java  |   5 -
 .../carbondata/core/datastore/ColumnType.java   |  51 
 .../core/datastore/DimensionType.java   |  35 ---
 .../carbondata/core/datastore/TableSpec.java| 116 ++---
 .../core/datastore/block/SegmentProperties.java |   4 +-
 .../datastore/chunk/AbstractRawColumnChunk.java |  10 +-
 .../chunk/impl/DimensionRawColumnChunk.java |   4 +-
 .../chunk/impl/MeasureRawColumnChunk.java   |   4 +-
 ...mpressedDimensionChunkFileBasedReaderV1.java |  10 +-
 ...mpressedDimensionChunkFileBasedReaderV2.java |  18 +-
 ...mpressedDimensionChunkFileBasedReaderV3.java |  24 +-
 .../measure/AbstractMeasureChunkReader.java |   6 +-
 ...CompressedMeasureChunkFileBasedReaderV1.java |  12 +-
 ...CompressedMeasureChunkFileBasedReaderV2.java |  18 +-
 ...CompressedMeasureChunkFileBasedReaderV3.java |  22 +-
 .../chunk/store/ColumnPageWrapper.java  |   6 +-
 .../core/datastore/page/ColumnPage.java | 174 +++--
 .../core/datastore/page/LazyColumnPage.java |   5 +-
 .../datastore/page/SafeFixLengthColumnPage.java |   7 +-
 .../datastore/page/SafeVarLengthColumnPage.java |  19 +-
 .../page/UnsafeFixLengthColumnPage.java |   7 +-
 .../page/UnsafeVarLengthColumnPage.java |  16 +-
 .../datastore/page/VarLengthColumnPageBase.java |  36 +--
 .../page/encoding/ColumnPageEncoder.java|  14 +-
 .../page/encoding/ColumnPageEncoderMeta.java|  76 --
 .../page/encoding/DefaultEncodingFactory.java   | 250 +++
 .../page/encoding/DefaultEncodingStrategy.java  | 243 --
 .../page/encoding/EncodingFactory.java  | 159 
 .../page/encoding/EncodingStrategy.java | 159 
 .../page/encoding/EncodingStrategyFactory.java  |  33 ---
 .../page/encoding/adaptive/AdaptiveCodec.java   |   6 -
 .../adaptive/AdaptiveDeltaIntegralCodec.java|  15 +-
 .../AdaptiveDeltaIntegralEncoderMeta.java   |  47 
 .../encoding/adaptive/AdaptiveEncoderMeta.java  |  69 -
 .../adaptive/AdaptiveFloatingCodec.java |  15 +-
 .../adaptive/AdaptiveFloatingEncoderMeta.java   |  47 
 .../adaptive/AdaptiveIntegralCodec.java |  15 +-
 .../adaptive/AdaptiveIntegralEncoderMeta.java   |  47 
 .../encoding/compress/DirectCompressCodec.java  |  24 +-
 .../compress/DirectCompressorEncoderMeta.java   |  57 -
 .../datastore/page/encoding/rle/RLECodec.java   |  17 +-
 .../page/encoding/rle/RLEEncoderMeta.java   |   6 +-
 .../statistics/PrimitivePageStatsCollector.java |   8 +-
 .../core/scan/complextypes/ArrayQueryType.java  |   7 +-
 .../scan/complextypes/ComplexQueryType.java |  15 +-
 .../scan/complextypes/PrimitiveQueryType.java   |   9 +-
 .../core/scan/complextypes/StructQueryType.java |   3 +-
 .../datastore/page/encoding/RLECodecSuite.java  |  10 +-
 .../core/util/CarbonMetadataUtilTest.java   |   5 +-
 examples/spark2/src/main/resources/data.csv |   1 +
 .../examples/CarbonSessionExample.scala |   3 +-
 .../TestComplexTypeWithBigArray.scala   | 160 
 .../execution/CarbonLateDecodeStrategy.scala|   4 +-
 .../processing/datatypes/ArrayDataType.java |  11 +
 .../processing/datatypes/GenericDataType.java   |   4 +
 .../processing/datatypes/PrimitiveDataType.java |  16 +-
 .../processing/datatypes/StructDataType.java|  15 ++
 .../carbondata/processing/store/TablePage.java  |  52 ++--
 .../util/CarbonDataProcessorUtil.java   |   6 +-
 59 files changed, 1136 insertions(+), 1101 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/8c1ddbf2/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
--
diff --git 

[13/51] [abbrv] carbondata git commit: [CARBONDATA-1458] Fixed backward compatibility issue with decimal

[ravipesala]

[CARBONDATA-1458] Fixed backward compatibility issue with decimal

The table loaded in 1.1 version cannot be queried in 1.2 branch as decimal 
min/max are hard coded to double.

This closes #1343


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/590bbb9b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/590bbb9b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/590bbb9b

Branch: refs/heads/branch-1.2
Commit: 590bbb9b65efa3c801f677113fd05b24ab2d218b
Parents: 4030cfb
Author: Ravindra Pesala 
Authored: Sat Sep 9 11:06:12 2017 +0530
Committer: Ravindra Pesala 
Committed: Mon Sep 11 12:03:27 2017 +0530

--
 .../java/org/apache/carbondata/core/util/CarbonUtil.java| 6 +++---
 .../carbondata/hive/CarbonDictionaryDecodeReadSupport.java  | 2 +-
 .../org/apache/carbondata/presto/CarbondataPageSource.java  | 8 +---
 .../org/apache/carbondata/presto/CarbondataRecordSet.java   | 2 --
 .../carbondata/presto/readers/DecimalSliceStreamReader.java | 4 +++-
 .../cluster/sdv/generated/AlterTableTestCase.scala  | 4 ++--
 .../sdv/generated/QueriesExcludeDictionaryTestCase.scala| 2 +-
 .../scala/org/apache/spark/sql/common/util/QueryTest.scala  | 9 +
 .../allqueries/InsertIntoCarbonTableTestCase.scala  | 9 -
 .../apache/carbondata/lcm/locks/ZooKeeperLockingTest.java   | 3 ++-
 10 files changed, 26 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/590bbb9b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
--
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java 
b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 8b6e44a..683633f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1452,9 +1452,9 @@ public final class CarbonUtil {
 valueEncoderMeta.setUniqueValue(buffer.getDouble());
 break;
   case CarbonCommonConstants.BIG_DECIMAL_MEASURE:
-valueEncoderMeta.setMaxValue(0.0);
-valueEncoderMeta.setMinValue(0.0);
-valueEncoderMeta.setUniqueValue(0.0);
+valueEncoderMeta.setMaxValue(BigDecimal.valueOf(Long.MAX_VALUE));
+valueEncoderMeta.setMinValue(BigDecimal.valueOf(Long.MIN_VALUE));
+valueEncoderMeta.setUniqueValue(BigDecimal.valueOf(Long.MIN_VALUE));
 break;
   case CarbonCommonConstants.BIG_INT_MEASURE:
 valueEncoderMeta.setMaxValue(buffer.getLong());

http://git-wip-us.apache.org/repos/asf/carbondata/blob/590bbb9b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
--
diff --git 
a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
 
b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
index b0adc69..76597fd 100644
--- 
a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
+++ 
b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
@@ -230,7 +230,7 @@ public class CarbonDictionaryDecodeReadSupport 
implements CarbonReadSupporthttp://git-wip-us.apache.org/repos/asf/carbondata/blob/590bbb9b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
--
diff --git 
a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
 
b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
index f13fb09..4520476 100644
--- 
a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
+++ 
b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
@@ -101,14 +101,13 @@ class CarbondataPageSource implements ConnectorPageSource 
{
 if (nanoStart == 0) {
   nanoStart = System.nanoTime();
 }
-Object vectorBatch;
 ColumnarBatch columnarBatch = null;
 int batchSize = 0;
 try {
   batchId++;
   if(vectorReader.nextKeyValue()) {
-vectorBatch = vectorReader.getCurrentValue();
-if(vectorBatch instanceof ColumnarBatch)
+Object vectorBatch = vectorReader.getCurrentValue();
+if(vectorBatch != null && vectorBatch instanceof ColumnarBatch)
 {
   columnarBatch = (ColumnarBatch) vectorBatch;
   batchSize = columnarBatch.numRows();
@@ -122,6 +121,9 @@ class CarbondataPageSource 

[45/51] [abbrv] carbondata git commit: [CARBONDATA-1438] Unify the sort column and sort scope in create table command

[ravipesala]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3cb6f65/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
--
diff --git 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
index 8f1369b..36b7b6e 100644
--- 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
+++ 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/GlobalSortTestCase.scala
@@ -183,8 +183,8 @@ class GlobalSortTestCase extends QueryTest with 
BeforeAndAfterAll {
   //Carbon-Loading-Optimizations-Global-Sort-01-01-14
   test("Carbon-Loading-Optimizations-Global-Sort-01-01-14", Include) {
 sql(
-  s"""CREATE TABLE uniqdata20c (CUST_ID int,CUST_NAME 
String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 
bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 
decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 
int) STORED BY 'carbondata' 
TBLPROPERTIES('NO_INVERTED_INDEX'='CUST_NAME')""").collect
-sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' 
into table uniqdata20c OPTIONS('SORT_SCOPE'='GLOBAL_SORT', 
'GLOBAL_SORT_PARTITIONS'='2','DELIMITER'=',' 
,'QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+  s"""CREATE TABLE uniqdata20c (CUST_ID int,CUST_NAME 
String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 
bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 
decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 
int) STORED BY 'carbondata' 
TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT','NO_INVERTED_INDEX'='CUST_NAME')""").collect
+sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' 
into table uniqdata20c OPTIONS('GLOBAL_SORT_PARTITIONS'='2','DELIMITER'=',' 
,'QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
 
 sql(s"""select * from uniqdata20c""").collect
 sql(s"""drop table if exists uniqdata20c""").collect
@@ -194,9 +194,9 @@ class GlobalSortTestCase extends QueryTest with 
BeforeAndAfterAll {
   //Carbon-Loading-Optimizations-Global-Sort-01-01-15
   test("Carbon-Loading-Optimizations-Global-Sort-01-01-15", Include) {
 sql(s"""drop table if exists t3""").collect
-sql(s"""CREATE TABLE t3 (ID Int, country String, name String, phonetype 
String, serialname String, salary Int,floatField float) STORED BY 
'carbondata'""").collect
+sql(s"""CREATE TABLE t3 (ID Int, country String, name String, phonetype 
String, serialname String, salary Int,floatField float) STORED BY 'carbondata' 
TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')""").collect
 sql(
-s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/batchsort/data.csv' into 
table t3 options('SINGLE_PASS'='TRUE','SORT_SCOPE'='GLOBAL_SORT', 
'GLOBAL_SORT_PARTITIONS'='2','COLUMNDICT'='country:$resourcesPath/Data/columndict/country.csv')""".stripMargin).collect
+s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/batchsort/data.csv' into 
table t3 options('SINGLE_PASS'='TRUE', 
'GLOBAL_SORT_PARTITIONS'='2','COLUMNDICT'='country:$resourcesPath/Data/columndict/country.csv')""".stripMargin).collect
 
 sql(s"""select * from t3""").collect
 sql(s"""drop table if exists t3""").collect
@@ -206,8 +206,8 @@ class GlobalSortTestCase extends QueryTest with 
BeforeAndAfterAll {
   //Carbon-Loading-Optimizations-Global-Sort-01-01-16
   test("Carbon-Loading-Optimizations-Global-Sort-01-01-16", Include) {
 sql(s"""drop table if exists t3""").collect
-sql(s"""CREATE TABLE t3 (ID Int, country String, name String, phonetype 
String, serialname String, salary Int,floatField float) STORED BY 
'carbondata'""").collect
-sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/batchsort/data.csv' 
into table t3 options('SINGLE_PASS'='TRUE','SORT_SCOPE'='GLOBAL_SORT', 
'GLOBAL_SORT_PARTITIONS'='2','ALL_DICTIONARY_PATH'='$resourcesPath/Data/columndict/data.dictionary')""").collect
+sql(s"""CREATE TABLE t3 (ID Int, country String, name String, phonetype 
String, serialname String, salary 

[51/51] [abbrv] carbondata git commit: [CARBONDATA-1316] Support drop partition function

[ravipesala]

[CARBONDATA-1316] Support drop partition function

This closes #1317


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/cb51b862
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/cb51b862
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/cb51b862

Branch: refs/heads/branch-1.2
Commit: cb51b86218cd815167f7c702b643ed0852c7f3dc
Parents: fe36e3b
Author: lionelcao 
Authored: Mon Sep 4 15:38:44 2017 +0800
Committer: QiangCai 
Committed: Mon Sep 18 17:19:22 2017 +0800

--
 .../core/metadata/schema/PartitionInfo.java |   5 +
 .../hadoop/api/CarbonTableInputFormat.java  |  23 ++-
 .../spark/partition/DropPartitionCallable.java  |  39 +
 .../org/apache/carbondata/spark/KeyVal.scala|   4 +-
 .../spark/rdd/AlterTableLoadPartitionRDD.scala  | 141 +++
 .../spark/rdd/AlterTableSplitPartitionRDD.scala | 146 
 .../spark/rdd/CarbonScanPartitionRDD.scala  |  29 ++--
 .../apache/carbondata/spark/rdd/Compactor.scala |   3 +-
 .../spark/rdd/DataManagementFunc.scala  |  50 +++---
 .../carbondata/spark/rdd/PartitionDropper.scala | 122 +
 .../spark/rdd/PartitionSplitter.scala   |  36 ++--
 .../carbondata/spark/util/CommonUtil.scala  |   2 +-
 .../spark/util/GlobalDictionaryUtil.scala   |   3 +-
 .../command/carbonTableSchemaCommon.scala   |  25 ++-
 .../org/apache/spark/util/PartitionUtils.scala  |  15 +-
 .../spark/rdd/CarbonDataRDDFactory.scala|   8 +-
 .../execution/command/carbonTableSchema.scala   |   2 +-
 .../spark/rdd/CarbonDataRDDFactory.scala| 103 ---
 .../execution/command/carbonTableSchema.scala   | 145 +++-
 .../sql/parser/CarbonSpark2SqlParser.scala  |  16 +-
 .../partition/TestAlterPartitionTable.scala | 171 +++
 .../processing/merger/CarbonDataMergerUtil.java |   5 +-
 .../processing/spliter/RowResultProcessor.java  | 105 
 .../spliter/RowResultSpliterProcessor.java  | 105 
 .../exception/AlterPartitionSliceException.java |  78 +
 .../exception/SliceSpliterException.java|  78 -
 26 files changed, 978 insertions(+), 481 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/cb51b862/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
index 4b0bc3e..d0c4447 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
@@ -92,6 +92,11 @@ public class PartitionInfo implements Serializable {
 numPartitions = numPartitions - 1 + newPartitionNumbers;
   }
 
+  public void dropPartition(int index) {
+partitionIds.remove(index);
+numPartitions--;
+  }
+
   public List getColumnSchemaList() {
 return columnSchemaList;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/cb51b862/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index dcc75bd..9076233 100644
--- 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -306,7 +306,7 @@ public class CarbonTableInputFormat extends 
FileInputFormat {
 // prune partitions for filter query on partition table
 BitSet matchedPartitions = null;
 if (partitionInfo != null) {
-  matchedPartitions = setMatchedPartitions(null, filter, partitionInfo);
+  matchedPartitions = setMatchedPartitions(null, filter, partitionInfo, 
null);
   if (matchedPartitions != null) {
 if (matchedPartitions.cardinality() == 0) {
   return new ArrayList();
@@ -366,9 +366,11 @@ public class CarbonTableInputFormat extends 
FileInputFormat {
   TableProvider tableProvider = new SingleTableProvider(carbonTable);
   // prune partitions for filter query on partition table
   String partitionIds = job.getConfiguration().get(ALTER_PARTITION_ID);
+  // matchedPartitions records partitionIndex, not partitionId
   BitSet matchedPartitions = null;
   if (partitionInfo != null) {
-matchedPartitions = setMatchedPartitions(partitionIds, 

[37/51] [abbrv] carbondata git commit: [CARBONDATA-1291]:carbonData query performance improvement when number of carbon blocks are high

[ravipesala]

[CARBONDATA-1291]:carbonData query performance improvement when number of 
carbon blocks are high

Limit query performance is slow when one load is having around 8400 carbondata 
files using Spark Distribution
This issue came, when number of blocks are high in that case for each block it 
is listing the delete delta file which is a expensive operation.
Solution- if IUD is done then only check for delete delta files

This closes #1324


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/940f4d5e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/940f4d5e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/940f4d5e

Branch: refs/heads/branch-1.2
Commit: 940f4d5e62c05bd47b192690c67b4970cad38466
Parents: 642b4bf
Author: kushalsaha 
Authored: Tue Sep 5 17:10:30 2017 +0530
Committer: Ravindra Pesala 
Committed: Fri Sep 15 09:44:38 2017 +0530

--
 .../carbondata/hadoop/CarbonInputFormat.java | 19 ++-
 .../hadoop/api/CarbonTableInputFormat.java   | 19 ++-
 2 files changed, 20 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/940f4d5e/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index fb3a637..4e8591e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -460,21 +460,22 @@ public class CarbonInputFormat extends 
FileInputFormat {
   for (DataRefNode dataRefNode : dataRefNodes) {
 BlockBTreeLeafNode leafNode = (BlockBTreeLeafNode) dataRefNode;
 TableBlockInfo tableBlockInfo = leafNode.getTableBlockInfo();
+String[] deleteDeltaFilePath = null;
 if (isIUDTable) {
   // In case IUD is not performed in this table avoid searching for
   // invalidated blocks.
   if (CarbonUtil
-  .isInvalidTableBlock(tableBlockInfo.getSegmentId(), 
tableBlockInfo.getFilePath(),
-  invalidBlockVOForSegmentId, updateStatusManager)) {
+  .isInvalidTableBlock(tableBlockInfo.getSegmentId(), 
tableBlockInfo.getFilePath(),
+  invalidBlockVOForSegmentId, updateStatusManager)) {
 continue;
   }
-}
-String[] deleteDeltaFilePath = null;
-try {
-  deleteDeltaFilePath =
-  
updateStatusManager.getDeleteDeltaFilePath(tableBlockInfo.getFilePath());
-} catch (Exception e) {
-  throw new IOException(e);
+  // When iud is done then only get delete delta files for a block
+  try {
+deleteDeltaFilePath =
+
updateStatusManager.getDeleteDeltaFilePath(tableBlockInfo.getFilePath());
+  } catch (Exception e) {
+throw new IOException(e);
+  }
 }
 result.add(new CarbonInputSplit(segmentNo, new 
Path(tableBlockInfo.getFilePath()),
 tableBlockInfo.getBlockOffset(), tableBlockInfo.getBlockLength(),

http://git-wip-us.apache.org/repos/asf/carbondata/blob/940f4d5e/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index f271517..dcc75bd 100644
--- 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -448,21 +448,22 @@ public class CarbonTableInputFormat extends 
FileInputFormat {
 invalidBlockVOForSegmentId =
 
updateStatusManager.getInvalidTimestampRange(inputSplit.getSegmentId());
   }
+  String[] deleteDeltaFilePath = null;
   if (isIUDTable) {
 // In case IUD is not performed in this table avoid searching for
 // invalidated blocks.
 if (CarbonUtil
-.isInvalidTableBlock(inputSplit.getSegmentId(), 
inputSplit.getPath().toString(),
-invalidBlockVOForSegmentId, updateStatusManager)) {
+.isInvalidTableBlock(inputSplit.getSegmentId(), 
inputSplit.getPath().toString(),
+invalidBlockVOForSegmentId, updateStatusManager)) {
   continue;
 }
-  }
-  String[] deleteDeltaFilePath = 

[49/51] [abbrv] carbondata git commit: [CARBONDATA-1478] Update compaction documentation

[ravipesala]

[CARBONDATA-1478] Update compaction documentation

In order to easier to understand, update the description about compaction.

This closes #1357


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/fe36e3bc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/fe36e3bc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/fe36e3bc

Branch: refs/heads/branch-1.2
Commit: fe36e3bc979958cf94bcefaeaa651903017ce084
Parents: c257468
Author: xubo245 <601450...@qq.com>
Authored: Wed Sep 13 20:24:40 2017 +0800
Committer: Jacky Li 
Committed: Mon Sep 18 17:10:59 2017 +0800

--
 docs/data-management.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/fe36e3bc/docs/data-management.md
--
diff --git a/docs/data-management.md b/docs/data-management.md
index 81866a1..b1a3eef 100644
--- a/docs/data-management.md
+++ b/docs/data-management.md
@@ -106,7 +106,7 @@ Example :
 CLEAN FILES FOR TABLE table1
 ```
 
- This DML will physically delete the segment which are "Marked for delete" 
immediately.
+ This DML will physically delete the segment which are "Marked for delete" and 
"Compacted" immediately.
 
 ## Compacting Data
   
@@ -135,7 +135,7 @@ CLEAN FILES FOR TABLE table1

 | Parameter | Default | Application | Description | Valid Values |
 
|-|-|-||--|
-| carbon.compaction.level.threshold | 4, 3 | Minor | This property is for 
minor compaction which decides how many segments to be merged. Example: If it 
is set as 2, 3 then minor compaction will be triggered for every 2 segments. 3 
is the number of level 1 compacted segment which is further compacted to new 
segment. | NA |
+| carbon.compaction.level.threshold | 4, 3 | Minor | This property is for 
minor compaction which decides how many segments to be merged. Example: If it 
is set as "2, 3", then minor compaction will be triggered for every 2 segments 
in level 1. 3 is the number of level 1 compacted segment which is further 
compacted to new segment in level 2. | NA |
 | carbon.major.compaction.size | 1024 MB | Major | Major compaction size can 
be configured using this parameter. Sum of the segments which is below this 
threshold will be merged. | NA |
 | carbon.numberof.preserve.segments | 0 | Minor/Major | This property 
configures number of segments to preserve from being compacted. Example: 
carbon.numberof.preserve.segments=2 then 2 latest segments will always be 
excluded from the compaction. No segments will be preserved by default. | 0-100 
|
 | carbon.allowed.compaction.days | 0 | Minor/Major | Compaction will merge the 
segments which are loaded within the specific number of days configured. 
Example: If the configuration is 2, then the segments which are loaded in the 
time frame of 2 days only will get merged. Segments which are loaded 2 days 
apart will not be merged. This is disabled by default. | 0-100 |

[08/51] [abbrv] carbondata git commit: [CARBONDATA-1399]Enable findbugs

[ravipesala]

[CARBONDATA-1399]Enable findbugs

This closes #1272


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1852e135
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1852e135
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1852e135

Branch: refs/heads/branch-1.2
Commit: 1852e135ae07a343b1f2a270e20d21069bd23c27
Parents: 0ebdc94
Author: Raghunandan S 
Authored: Sat Aug 19 21:34:39 2017 +0530
Committer: Ravindra Pesala 
Committed: Sat Sep 9 18:29:11 2017 +0530

--
 .../generator/key/DictionaryMessage.java|  6 ++-
 pom.xml | 50 
 2 files changed, 25 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/1852e135/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
 
b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
index 749c3f4..d59e9f8 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
@@ -55,7 +55,8 @@ public class DictionaryMessage {
   public void readData(ByteBuf byteBuf) {
 byte[] tableIdBytes = new byte[byteBuf.readInt()];
 byteBuf.readBytes(tableIdBytes);
-tableUniqueId = new String(tableIdBytes);
+tableUniqueId =
+new String(tableIdBytes, 
Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
 
 byte[] colBytes = new byte[byteBuf.readInt()];
 byteBuf.readBytes(colBytes);
@@ -79,7 +80,8 @@ public class DictionaryMessage {
 // Just reserve the bytes to add length of header at last.
 byteBuf.writeShort(Short.MAX_VALUE);
 
-byte[] tableIdBytes = tableUniqueId.getBytes();
+byte[] tableIdBytes =
+
tableUniqueId.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
 byteBuf.writeInt(tableIdBytes.length);
 byteBuf.writeBytes(tableIdBytes);
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1852e135/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 3540221..a1301f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,6 @@
 hadoop
 integration/spark-common
 integration/spark-common-test
-integration/hive
 assembly
   
 
@@ -329,6 +328,27 @@
 
   
   
+org.codehaus.mojo
+findbugs-maven-plugin
+3.0.4
+
+  
${dev.path}/findbugs-exclude.xml
+  true
+  true
+  true
+  Max
+
+
+  
+analyze-compile
+compile
+
+  check
+
+  
+
+  
+  
 org.jacoco
 jacoco-maven-plugin
 0.7.9
@@ -556,34 +576,6 @@
   
 
 
-  findbugs
-  
-
-  
-org.codehaus.mojo
-findbugs-maven-plugin
-3.0.4
-
-  
${dev.path}/findbugs-exclude.xml
-  true
-  true
-  true
-  Max
-
-
-  
-analyze-compile
-compile
-
-  check
-
-  
-
-  
-
-  
-
-
   include-all
 
 

[32/51] [abbrv] carbondata git commit: [CARBONDATA-1472] Optimize memory and fix nosort queries

[ravipesala]

[CARBONDATA-1472] Optimize memory and fix nosort queries

1.Use UnsafeManager for dimension chunks as well to avoid leaks
2.Fix filters on nosort columns.
3.Optimize scanRDD

This closes #1346


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/887310fc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/887310fc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/887310fc

Branch: refs/heads/branch-1.2
Commit: 887310fc75e8c20c82929d2d92114887cecf44df
Parents: dde2f4c
Author: Ravindra Pesala 
Authored: Sun Sep 10 14:57:09 2017 +0530
Committer: Jacky Li 
Committed: Wed Sep 13 22:03:26 2017 +0800

--
 .../core/constants/CarbonCommonConstants.java   |  4 ++
 .../UnsafeAbstractDimensionDataChunkStore.java  | 17 +---
 .../core/memory/MemoryAllocatorFactory.java | 46 
 .../core/memory/UnsafeMemoryManager.java| 21 ++---
 .../executor/impl/AbstractQueryExecutor.java|  6 +--
 .../executer/RangeValueFilterExecuterImpl.java  | 10 +++--
 ...velRangeLessThanEqualFilterExecuterImpl.java |  8 +++-
 .../RowLevelRangeLessThanFiterExecuterImpl.java |  8 +++-
 .../carbondata/hadoop/AbstractRecordReader.java |  2 -
 .../carbondata/spark/rdd/CarbonScanRDD.scala| 43 +-
 10 files changed, 84 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/887310fc/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 5a68f60..0348bd1 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1371,6 +1371,10 @@ public final class CarbonCommonConstants {
 
   public static final String USE_DISTRIBUTED_DATAMAP_DEFAULT = "false";
 
+  public static final String CARBON_USE_BLOCKLET_DISTRIBUTION = 
"carbon.blocklet.distribution";
+
+  public static final String CARBON_USE_BLOCKLET_DISTRIBUTION_DEFAULT = "true";
+
   private CarbonCommonConstants() {
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/887310fc/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
index 704f2d3..22c2e16 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
@@ -20,9 +20,11 @@ package 
org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import 
org.apache.carbondata.core.datastore.chunk.store.DimensionDataChunkStore;
 import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
 import org.apache.carbondata.core.memory.MemoryBlock;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.memory.UnsafeMemoryManager;
 import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
 
 /**
  * Responsibility is to store dimension data in memory. storage can be on heap
@@ -60,6 +62,8 @@ public abstract class UnsafeAbstractDimensionDataChunkStore 
implements Dimension
*/
   protected boolean isMemoryOccupied;
 
+  private final long taskId = 
ThreadLocalTaskInfo.getCarbonTaskInfo().getTaskId();
+
   /**
* Constructor
*
@@ -69,9 +73,12 @@ public abstract class UnsafeAbstractDimensionDataChunkStore 
implements Dimension
*/
   public UnsafeAbstractDimensionDataChunkStore(long totalSize, boolean 
isInvertedIdex,
   int numberOfRows) {
-// allocating the data page
-this.dataPageMemoryBlock =
-
MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(totalSize);
+try {
+  // allocating the data page
+  this.dataPageMemoryBlock = 
UnsafeMemoryManager.allocateMemoryWithRetry(taskId, totalSize);
+} catch (MemoryException e) {
+  throw new RuntimeException(e);
+}
 

[15/51] [abbrv] carbondata git commit: [CARBONDATA-1462]Add an option 'carbon.update.storage.level' to support configuring the storage level when updating data with 'carbon.update.persist.enable'='tru

[ravipesala]

[CARBONDATA-1462]Add an option 'carbon.update.storage.level' to support 
configuring the storage level when updating data with 
'carbon.update.persist.enable'='true'

When updating data with 'carbon.update.persist.enable'='true'(default), the 
storage level of dataset is 'MEMORY_AND_DISK', it should support configuring 
the storage level to correspond to different environment.

This closes #1340


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0ab928e9
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0ab928e9
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0ab928e9

Branch: refs/heads/branch-1.2
Commit: 0ab928e9c1730d69a3fcd1805c26ef1200214fc9
Parents: 8b38e0b
Author: Zhang Zhichao <441586...@qq.com>
Authored: Fri Sep 8 13:27:42 2017 +0800
Committer: Ravindra Pesala 
Committed: Mon Sep 11 20:33:57 2017 +0530

--
 .../core/constants/CarbonCommonConstants.java   | 20 +++
 .../carbondata/core/util/CarbonProperties.java  | 36 
 .../sql/execution/command/IUDCommands.scala | 17 ++---
 3 files changed, 59 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ab928e9/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 6c116a7..5a68f60 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1091,6 +1091,26 @@ public final class CarbonCommonConstants {
   public static final String defaultValueIsPersistEnabled = "true";
 
   /**
+   * Which storage level to persist dataset when updating data
+   * with 'carbon.update.persist.enable'='true'
+   */
+  @CarbonProperty
+  public static final String CARBON_UPDATE_STORAGE_LEVEL =
+  "carbon.update.storage.level";
+
+  /**
+   * The default value(MEMORY_AND_DISK) is the same as the default storage 
level of Dataset.
+   * Unlike `RDD.cache()`, the default storage level is set to be 
`MEMORY_AND_DISK` because
+   * recomputing the in-memory columnar representation of the underlying table 
is expensive.
+   *
+   * if user's executor has less memory, set the CARBON_UPDATE_STORAGE_LEVEL
+   * to MEMORY_AND_DISK_SER or other storage level to correspond to different 
environment.
+   * You can get more recommendations about storage level in spark website:
+   * 
http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence.
+   */
+  public static final String CARBON_UPDATE_STORAGE_LEVEL_DEFAULT = 
"MEMORY_AND_DISK";
+
+  /**
* current data file version
*/
   public static final String CARBON_DATA_FILE_DEFAULT_VERSION = "V3";

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ab928e9/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java 
b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 4e9c21a..0ab28e2 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -890,6 +890,42 @@ public final class CarbonProperties {
   }
 
   /**
+   * Return valid CARBON_UPDATE_STORAGE_LEVEL
+   * @return boolean
+   */
+  public boolean isPersistUpdateDataset() {
+String isPersistEnabled = 
getProperty(CarbonCommonConstants.isPersistEnabled,
+CarbonCommonConstants.defaultValueIsPersistEnabled);
+boolean validatePersistEnabled = 
CarbonUtil.validateBoolean(isPersistEnabled);
+if (!validatePersistEnabled) {
+  LOGGER.error("The " + CarbonCommonConstants.isPersistEnabled
+  + " configuration value is invalid. It will use default value("
+  + CarbonCommonConstants.defaultValueIsPersistEnabled
+  + ").");
+  isPersistEnabled = CarbonCommonConstants.defaultValueIsPersistEnabled;
+}
+return isPersistEnabled.equalsIgnoreCase("true");
+  }
+
+  /**
+   * Return valid storage level for CARBON_UPDATE_STORAGE_LEVEL
+   * @return String
+   */
+  public String getUpdateDatasetStorageLevel() {
+String storageLevel = 
getProperty(CarbonCommonConstants.CARBON_UPDATE_STORAGE_LEVEL,
+CarbonCommonConstants.CARBON_UPDATE_STORAGE_LEVEL_DEFAULT);
+boolean validateStorageLevel = 
CarbonUtil.isValidStorageLevel(storageLevel);
+if 

[09/51] [abbrv] carbondata git commit: [CARBONDATA-1463] CompareTest should validate result size

[ravipesala]

[CARBONDATA-1463] CompareTest should validate result size

CompareTest for spark2.1 should only validate result size instead of result 
value, because some test case include aggregation on double column which will 
give different result since carbon records are sorted

This closes #1341


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a5483e8c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a5483e8c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a5483e8c

Branch: refs/heads/branch-1.2
Commit: a5483e8c5a928fefaa130ac6ac5973ba459ae23f
Parents: 1852e13
Author: Jacky Li 
Authored: Fri Sep 8 22:06:41 2017 +0800
Committer: Ravindra Pesala 
Committed: Sat Sep 9 18:38:28 2017 +0530

--
 .../main/scala/org/apache/carbondata/examples/CompareTest.scala | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/a5483e8c/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
--
diff --git 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
index ffc4b22..52ccd5f 100644
--- 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
+++ 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
@@ -320,7 +320,10 @@ object CompareTest {
 
   private def printErrorIfNotMatch(index: Int, table1: String, result1: 
Array[Row],
   table2: String, result2: Array[Row]): Unit = {
-if (!result1.sameElements(result2)) {
+// check result size instead of result value, because some test case 
include
+// aggregation on double column which will give different result since 
carbon
+// records are sorted
+if (result1.length != result2.length) {
   val num = index + 1
   println(s"$table1 result for query $num: ")
   println(s"""${result1.mkString(",")}""")

[23/51] [abbrv] carbondata git commit: [CARBONDATA-1413]Validate for invalid range info in partition definition

[ravipesala]

[CARBONDATA-1413]Validate for invalid range info in partition definition

This closes #1323


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9f0ac24d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9f0ac24d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9f0ac24d

Branch: refs/heads/branch-1.2
Commit: 9f0ac24d900549c0d88810e7d95007a0544ce086
Parents: b8ecf81
Author: Raghunandan S 
Authored: Tue Sep 5 16:58:31 2017 +0530
Committer: Ravindra Pesala 
Committed: Wed Sep 13 08:41:54 2017 +0530

--
 .../partition/TestDDLForPartitionTable.scala| 20 
 .../carbondata/spark/util/CommonUtil.scala  |  4 
 2 files changed, 24 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f0ac24d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
index 8083fde..8c79398 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
@@ -367,6 +367,25 @@ class TestDDLForPartitionTable  extends QueryTest with 
BeforeAndAfterAll {
 assert(exception_test_range_decimal.getMessage.contains("Invalid Partition 
Values"))
   }
 
+  test("Invalid Partition Range") {
+val exceptionMessage: Exception = intercept[Exception] {
+  sql(
+"""
+  | CREATE TABLE default.rangeTableInvalid (empno int, empname String, 
designation String,
+  |  workgroupcategory int, workgroupcategoryname String, deptno int, 
deptname String,
+  |  projectcode int, projectjoindate Timestamp, projectenddate 
Timestamp,attendance int,
+  |  utilization int,salary int)
+  | PARTITIONED BY (doj Timestamp)
+  | STORED BY 'org.apache.carbondata.format'
+  | TBLPROPERTIES('PARTITION_TYPE'='RANGE',
+  |  'RANGE_INFO'='2017-06-11 00:00:02')
+""".stripMargin)
+}
+
+assert(exceptionMessage.getMessage
+  .contains("Range info must define a valid range.Please check again!"))
+  }
+
   override def afterAll = {
 dropTable
   }
@@ -395,6 +414,7 @@ class TestDDLForPartitionTable  extends QueryTest with 
BeforeAndAfterAll {
 sql("DROP TABLE IF EXISTS test_range_date")
 sql("DROP TABLE IF EXISTS test_range_timestamp")
 sql("DROP TABLE IF EXISTS test_range_decimal")
+sql("DROP TABLE IF EXISTS rangeTableInvalid")
   }
 
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f0ac24d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index c67806e..ed4d784 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -323,6 +323,10 @@ object CommonUtil {
*/
   def validateRangeInfo(rangeInfo: List[String], columnDataType: DataType,
   timestampFormatter: SimpleDateFormat, dateFormatter: SimpleDateFormat): 
Unit = {
+if (rangeInfo.size <= 1) {
+  throw new
+ MalformedCarbonCommandException("Range info must define a valid 
range.Please check again!")
+}
 val comparator = Comparator.getComparator(columnDataType)
 var head = columnDataType match {
   case DataType.STRING => ByteUtil.toBytes(rangeInfo.head)

[35/51] [abbrv] carbondata git commit: [CARBONDATA-1408]:Data loading with globalSort is failing in long run

[ravipesala]

[CARBONDATA-1408]:Data loading with globalSort is failing in long run

After 437 load, data loading getting failed.
On analyzing the root cause it concluded that any new folder is not getting 
created in temp location.
There's a certain limit based on OS in folder creation. Thus in long run data 
loading case the folder creation limit reaches its maximum due to which any new 
folder is not getting created in temp folder, and thus the data loading is 
getting failed.
Solution:- Need to delete the created folder from temp folder once data loading 
is completed.

This closes #1330


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b269384d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b269384d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b269384d

Branch: refs/heads/branch-1.2
Commit: b269384d4e88d3571ed964b04372f916c13b3f04
Parents: df95547
Author: kushalsaha 
Authored: Wed Sep 6 11:32:37 2017 +0530
Committer: Ravindra Pesala 
Committed: Thu Sep 14 16:42:25 2017 +0530

--
 .../carbondata/spark/load/DataLoadProcessorStepOnSpark.scala   | 2 +-
 .../scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala| 2 +-
 .../org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b269384d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index e43d204..aaf7926 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -225,7 +225,7 @@ object DataLoadProcessorStepOnSpark {
 } else {
   storeLocation = System.getProperty("java.io.tmpdir")
 }
-storeLocation = storeLocation + '/' + System.nanoTime() + '/' + index
+storeLocation = storeLocation + '/' + System.nanoTime() + '_' + index
 storeLocation
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b269384d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 52f37ef..c8a55af 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -101,7 +101,7 @@ class CarbonMergerRDD[K, V](
   } else {
 storeLocation = System.getProperty("java.io.tmpdir")
   }
-  storeLocation = storeLocation + '/' + System.nanoTime() + '/' + 
theSplit.index
+  storeLocation = storeLocation + '/' + System.nanoTime() + '_' + 
theSplit.index
   var mergeStatus = false
   var mergeNumber = ""
   var exec: CarbonCompactionExecutor = null

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b269384d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 7c6274b..67c71a0 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -169,7 +169,7 @@ class SparkPartitionLoader(model: CarbonLoadModel,
 LOGGER.info("Temp location for loading data: " + 
storeLocation.mkString(","))
   }
 
-  private def tmpLocationSuffix = File.separator + System.nanoTime() + 
File.separator + splitIndex
+  private def tmpLocationSuffix = File.separator + System.nanoTime() + "_" + 
splitIndex
 }
 
 /**

[39/51] [abbrv] carbondata git commit: [CARBONDATA-1348] Sort_Column should not supported for no-dictionary

[ravipesala]

[CARBONDATA-1348] Sort_Column should not supported for no-dictionary

Sort_Column should not supported for no dictionary column having numeric 
data-type and measure column.

This closes #1354


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/302ef2f5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/302ef2f5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/302ef2f5

Branch: refs/heads/branch-1.2
Commit: 302ef2f56f9c0b39a5b1d29fe25af3236a0ddb29
Parents: 8791eab
Author: rahulforallp 
Authored: Wed Sep 13 15:43:33 2017 +0530
Committer: Ravindra Pesala 
Committed: Fri Sep 15 14:00:25 2017 +0530

--
 .../testsuite/sortcolumns/TestSortColumns.scala | 20 ++--
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 12 
 2 files changed, 30 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/302ef2f5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 2704d23..bd1264a 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -53,7 +53,7 @@ class TestSortColumns extends QueryTest with 
BeforeAndAfterAll {
   }
 
   test(
-"create table with no dictionary sort_columns where 
NumberOfNoDictSortColumns < " +
+"create table with no dictionary sort_columns where 
NumberOfNoDictSortColumns is less than " +
 "NoDictionaryCount")
   {
 sql(
@@ -283,7 +283,7 @@ class TestSortColumns extends QueryTest with 
BeforeAndAfterAll {
 checkExistence(sql("describe formatted sorttableDesc"),true,"SORT_COLUMNS")
 checkExistence(sql("describe formatted 
sorttableDesc"),true,"empno,empname")
   }
-  
+
   test("duplicate columns in sort_columns") {
 val exceptionCaught = intercept[MalformedCarbonCommandException]{
   sql("CREATE TABLE sorttable1 (empno int, empname String, designation 
String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, 
deptno int, deptname String, projectcode int, projectjoindate Timestamp, 
projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 
'org.apache.carbondata.format' 
tblproperties('sort_columns'='empno,empname,empno')")
@@ -291,6 +291,21 @@ class TestSortColumns extends QueryTest with 
BeforeAndAfterAll {
   assert(exceptionCaught.getMessage.equals("SORT_COLUMNS Either having 
duplicate columns : empno or it contains illegal argumnet."))
   }
 
+  test("Measure columns in sort_columns") {
+val exceptionCaught = intercept[MalformedCarbonCommandException] {
+  sql(
+"CREATE TABLE sorttable1 (empno Double, empname String, designation 
String, doj Timestamp, " +
+"workgroupcategory int, workgroupcategoryname String, deptno int, 
deptname String, " +
+"projectcode int, projectjoindate Timestamp, projectenddate 
Timestamp,attendance int," +
+"utilization int,salary int) STORED BY 'org.apache.carbondata.format' 
tblproperties" +
+"('sort_columns'='empno')")
+}
+println(exceptionCaught.getMessage)
+assert(exceptionCaught.getMessage
+  .equals(
+"sort_columns is unsupported for double datatype column: empno"))
+  }
+
   override def afterAll = {
 dropTable
   }
@@ -319,6 +334,7 @@ class TestSortColumns extends QueryTest with 
BeforeAndAfterAll {
 sql("drop table if exists unsortedtable_heap_inmemory")
 sql("drop table if exists test_sort_col")
 sql("drop table if exists test_sort_col_hive")
+sql("drop table if exists sorttable1b")
   }
 
   def setLoadingProperties(offheap: String, unsafe: String, useBatch: String): 
Unit = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/302ef2f5/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 03aac20..661f724 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ 

[21/51] [abbrv] carbondata git commit: [CARBONDATA-1446] Fixed Bug for error message on invalid partition id in alter partition command

[ravipesala]

[CARBONDATA-1446] Fixed Bug for error message on invalid partition id in alter 
partition command

1.In alter partition command, if the user has given invalid partition id ( that 
is, partition id which does not exist ), this case is not handled, and the 
invalid partition id results in inappropriate exception further in code.
2.In this PR, an appropriate exception is thrown for Invalid Partition Id.
3.Added test case for the same in class TestAlterPartitionTable.scala

This closes #1320


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/92f34700
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/92f34700
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/92f34700

Branch: refs/heads/branch-1.2
Commit: 92f347008fbdcf2a80d1f8b0c7f9e316a2524127
Parents: fc39b28
Author: ksimar 
Authored: Mon Sep 4 18:13:32 2017 +0530
Committer: Jacky Li 
Committed: Tue Sep 12 22:11:34 2017 +0800

--
 .../scala/org/apache/spark/util/PartitionUtils.scala |  8 ++--
 .../testsuite/partition/TestAlterPartitionTable.scala| 11 +++
 2 files changed, 17 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/92f34700/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
 
b/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
index 184ab9e..3982f7b 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
@@ -80,12 +80,16 @@ object PartitionUtils {
   dateFormatter: SimpleDateFormat): Unit = {
 val columnDataType = partitionInfo.getColumnSchemaList.get(0).getDataType
 val index = partitionIdList.indexOf(partitionId)
+if (index < 0) {
+  throw new IllegalArgumentException("Invalid Partition Id " + partitionId 
+
+"\n Use show partitions table_name to get the list of valid 
partitions")
+}
 if (partitionInfo.getPartitionType == PartitionType.RANGE) {
   val rangeInfo = partitionInfo.getRangeInfo.asScala.toList
   val newRangeInfo = partitionId match {
 case 0 => rangeInfo ++ splitInfo
 case _ => rangeInfo.take(index - 1) ++ splitInfo ++
-  rangeInfo.takeRight(rangeInfo.size - index)
+  rangeInfo.takeRight(rangeInfo.size - index)
   }
   CommonUtil.validateRangeInfo(newRangeInfo, columnDataType,
 timestampFormatter, dateFormatter)
@@ -102,7 +106,7 @@ object PartitionUtils {
   val newListInfo = partitionId match {
 case 0 => originList ++ addListInfo
 case _ => originList.take(index - 1) ++ addListInfo ++
-  originList.takeRight(originList.size - index)
+  originList.takeRight(originList.size - index)
   }
   partitionInfo.setListInfo(newListInfo.map(_.asJava).asJava)
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/92f34700/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index 090a636..9de2ef5 100644
--- 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -342,6 +342,17 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 checkAnswer(result_after5, result_origin5)
   }
 
+  test("test exception if invalid partition id is provided in alter command") {
+sql("drop table if exists test_invalid_partition_id")
+
+sql("CREATE TABLE test_invalid_partition_id (CUST_NAME 
String,ACTIVE_EMUI_VERSION string,DOB Timestamp,DOJ timestamp, " +
+  "BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 
decimal(30,10), DECIMAL_COLUMN2 decimal(36,10)," +
+  "Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) 
PARTITIONED BY (CUST_ID int)" +
+  " STORED BY 'org.apache.carbondata.format' " +
+  "TBLPROPERTIES 
('PARTITION_TYPE'='RANGE','RANGE_INFO'='9090,9500,9800',\"TABLE_BLOCKSIZE\"= 
\"256 MB\")")
+intercept[IllegalArgumentException] { sql("ALTER TABLE 
test_invalid_partition_id SPLIT 

[12/51] [abbrv] carbondata git commit: [CARBONDATA-1379] Fixed Date range filter with cast not working

[ravipesala]

[CARBONDATA-1379] Fixed Date range filter with cast not working

This closes #1254


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4030cfb2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4030cfb2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4030cfb2

Branch: refs/heads/branch-1.2
Commit: 4030cfb27795e7d8dea6dadd7573bc0e3265a437
Parents: 252c3e3
Author: Ravindra Pesala 
Authored: Sat Aug 12 11:42:26 2017 +0530
Committer: Jacky Li 
Committed: Sun Sep 10 23:38:21 2017 +0800

--
 .../timestamp/DateDirectDictionaryGenerator.java   | 13 +++--
 .../core/scan/expression/ExpressionResult.java |  8 +++-
 .../DateDataTypeDirectDictionaryTest.scala | 11 +++
 3 files changed, 21 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/4030cfb2/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index 0d7cb6c..5a6e03d 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -18,7 +18,6 @@ package 
org.apache.carbondata.core.keygenerator.directdictionary.timestamp;
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
-import java.util.Calendar;
 import java.util.Date;
 import java.util.TimeZone;
 
@@ -37,16 +36,10 @@ public class DateDirectDictionaryGenerator implements 
DirectDictionaryGenerator
 
   private static final int cutOffDate = Integer.MAX_VALUE >> 1;
   private static final long SECONDS_PER_DAY = 60 * 60 * 24L;
-  private static final long MILLIS_PER_DAY = SECONDS_PER_DAY * 1000L;
+  public static final long MILLIS_PER_DAY = SECONDS_PER_DAY * 1000L;
 
   private ThreadLocal simpleDateFormatLocal = new 
ThreadLocal<>();
 
-  //Java TimeZone has no mention of thread safety. Use thread local instance 
to be safe.
-  private ThreadLocal threadLocalLocalTimeZone = new ThreadLocal() {
-@Override protected TimeZone initialValue() {
-  return Calendar.getInstance().getTimeZone();
-}
-  };
   private String dateFormat;
 
   /**
@@ -154,14 +147,14 @@ public class DateDirectDictionaryGenerator implements 
DirectDictionaryGenerator
   }
 
   private int generateKey(long timeValue) {
-long milli = timeValue + 
threadLocalLocalTimeZone.get().getOffset(timeValue);
-return (int) Math.floor((double) milli / MILLIS_PER_DAY) + cutOffDate;
+return (int) Math.floor((double) timeValue / MILLIS_PER_DAY) + cutOffDate;
   }
 
   public void initialize() {
 if (simpleDateFormatLocal.get() == null) {
   simpleDateFormatLocal.set(new SimpleDateFormat(dateFormat));
   simpleDateFormatLocal.get().setLenient(false);
+  simpleDateFormatLocal.get().setTimeZone(TimeZone.getTimeZone("GMT"));
 }
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4030cfb2/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
index 74e666b..08b1972 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
@@ -24,8 +24,10 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.TimeZone;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import 
org.apache.carbondata.core.keygenerator.directdictionary.timestamp.DateDirectDictionaryGenerator;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import 
org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
 import org.apache.carbondata.core.util.CarbonUtil;
@@ -177,6 +179,9 @@ public class ExpressionResult implements 
Comparable {
 case TIMESTAMP:
   String format = CarbonUtil.getFormatFromProperty(this.getDataType());
   SimpleDateFormat parser = new SimpleDateFormat(format);
+  if (this.getDataType() == 

[22/51] [abbrv] carbondata git commit: [DOC] Update installation-guide.md

[ravipesala]

[DOC] Update installation-guide.md

This closes #1174


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b8ecf817
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b8ecf817
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b8ecf817

Branch: refs/heads/branch-1.2
Commit: b8ecf8176b63de4aec4dfb421fd6ade3fab9eda6
Parents: 92f3470
Author: chenerlu 
Authored: Sat Jul 15 00:34:34 2017 +0800
Committer: chenliang613 
Committed: Tue Sep 12 23:19:30 2017 +0800

--
 docs/installation-guide.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8ecf817/docs/installation-guide.md
--
diff --git a/docs/installation-guide.md b/docs/installation-guide.md
index c7d694d..acb952a 100644
--- a/docs/installation-guide.md
+++ b/docs/installation-guide.md
@@ -182,9 +182,10 @@ hdfs://:port/user/hive/warehouse/carbon.store
 
 ```
  cd $SPARK_HOME
- ./bin/beeline jdbc:hive2://:port
+ ./sbin/start-thriftserver.sh
+ ./bin/beeline -u jdbc:hive2://:port
 
  Example
- ./bin/beeline jdbc:hive2://10.10.10.10:1
+ ./bin/beeline -u jdbc:hive2://10.10.10.10:1
 ```
 

[26/51] [abbrv] carbondata git commit: [CARBONDATA-1452] Issue with loading timestamp data beyond cutoff

[ravipesala]

[CARBONDATA-1452] Issue with loading timestamp data beyond cutoff

While generating surrogate for timestamp dictionary column, we are casting the 
value to int. We are considering only the +ve values for generating dictionary, 
when the value is out of range,overflow occurs and cyclic rotation happens 
while casting, in the cyclic rotation there is possibility of getting +ve 
values in overflow cases too.

Lets say cutoff timestamp is 1970-01-01 05:30:00, so we will be able to load 
data 68 years from this date, not beyond 68 years

While loading 3007-01-01 00:00:00, dictionary generation will throw bad record 
exception as converting this data to int is -ve (overflows and cyclic rotation)

But while loading 4016-01-01 00:00:00, dictionary will be generated for this as 
converting this data to int is +ve (overflows and cyclic rotation) --> This 
data is loaded but not as actual value. Different timestamp will be loaded.

This PR has,

(1) Refactoring
(2) Checking overflow

This closes #1335


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2176a2f1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2176a2f1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2176a2f1

Branch: refs/heads/branch-1.2
Commit: 2176a2f1d317763f5423dc1a5c254ee29e096c4b
Parents: a8b3fac
Author: dhatchayani 
Authored: Wed Sep 6 15:43:56 2017 +0530
Committer: Ravindra Pesala 
Committed: Wed Sep 13 15:34:36 2017 +0530

--
 .../TimeStampDirectDictionaryGenerator.java | 23 +---
 1 file changed, 10 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/2176a2f1/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index 6a0b9e6..c8b88d8 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -89,7 +89,7 @@ public class TimeStampDirectDictionaryGenerator implements 
DirectDictionaryGener
 }
 long cutOffTimeStampLocal;
 if (null == cutOffTimeStampString) {
-  cutOffTimeStampLocal = -1;
+  cutOffTimeStampLocal = 0;
 } else {
   try {
 SimpleDateFormat timeParser = new 
SimpleDateFormat(CarbonProperties.getInstance()
@@ -102,7 +102,7 @@ public class TimeStampDirectDictionaryGenerator implements 
DirectDictionaryGener
 LOGGER.warn("Cannot convert" + cutOffTimeStampString
 + " to Time/Long type value. Value considered for cutOffTimeStamp 
is -1." + e
 .getMessage());
-cutOffTimeStampLocal = -1;
+cutOffTimeStampLocal = 0;
   }
 }
 granularityFactor = granularityFactorLocal;
@@ -187,12 +187,7 @@ public class TimeStampDirectDictionaryGenerator implements 
DirectDictionaryGener
 if (key == 1) {
   return null;
 }
-long timeStamp = 0;
-if (cutOffTimeStamp >= 0) {
-  timeStamp = ((key - 2) * granularityFactor + cutOffTimeStamp);
-} else {
-  timeStamp = (key - 2) * granularityFactor;
-}
+long timeStamp = ((key - 2) * granularityFactor + cutOffTimeStamp);
 return timeStamp * 1000L;
   }
 
@@ -215,13 +210,15 @@ public class TimeStampDirectDictionaryGenerator 
implements DirectDictionaryGener
   }
 
   private int generateKey(long timeValue) {
-if (cutOffTimeStamp >= 0) {
-  int keyValue = (int) ((timeValue - cutOffTimeStamp) / granularityFactor);
-  return keyValue < 0 ? 1 : keyValue + 2;
-} else {
-  int keyValue = (int) (timeValue / granularityFactor);
+if (timeValue >= 0) {
+  long time = (timeValue - cutOffTimeStamp) / granularityFactor;
+  int keyValue = -1;
+  if (time <= (long) Integer.MAX_VALUE) {
+keyValue = (int) time;
+  }
   return keyValue < 0 ? 1 : keyValue + 2;
 }
+return 1;
   }
 
   public void initialize() {

[31/51] [abbrv] carbondata git commit: [CARBONDATA-1221] Documentation - removed the unused parameter

[ravipesala]

[CARBONDATA-1221] Documentation - removed the unused parameter

This closes #1282


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dde2f4cc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dde2f4cc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dde2f4cc

Branch: refs/heads/branch-1.2
Commit: dde2f4cc2a4d40c768d543d087ce4e40bf44dd4d
Parents: 8c1ddbf
Author: Ayushi93 
Authored: Wed Sep 13 20:53:54 2017 +0800
Committer: chenliang613 
Committed: Wed Sep 13 20:55:24 2017 +0800

--
 docs/configuration-parameters.md | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/dde2f4cc/docs/configuration-parameters.md
--
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index 8101aa1..77320f6 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -83,9 +83,7 @@ This section provides the details of all the configurations 
required for CarbonD
 | Parameter | Default Value | Description | Range |
 
|--|---|---|---|
 | carbon.number.of.cores | 4 | Number of cores to be used while querying. |  |
-| carbon.inmemory.record.size | 12 | Number of records to be in memory 
while querying. | Min=10 and Max=24 |
 | carbon.enable.quick.filter | false | Improves the performance of filter 
query. |  |
-| no.of.cores.to.load.blocks.in.driver | 10 | Number of core to load the 
blocks in driver. |  |
 
 
 ##   Miscellaneous Configuration

[06/51] [abbrv] carbondata git commit: [CARBONDATA-1464] Fixed SparkSessionExample

[ravipesala]

[CARBONDATA-1464] Fixed SparkSessionExample

Not able to create table from SparkSession because of missing tablePath. This 
PR generates tablePath from storelocation.

This closes #1342


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2d75c466
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2d75c466
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2d75c466

Branch: refs/heads/branch-1.2
Commit: 2d75c4661583d9765c11874ffc9dd804154b74ea
Parents: cd2332e
Author: Ravindra Pesala 
Authored: Fri Sep 8 21:20:18 2017 +0530
Committer: chenliang613 
Committed: Sat Sep 9 07:58:02 2017 +0800

--
 .../org/apache/spark/sql/CarbonSource.scala | 89 +++-
 1 file changed, 48 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/2d75c466/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala 
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index bec163b..1b021b0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -25,8 +25,8 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
 import org.apache.spark.sql.catalyst.catalog.CatalogTable
 import org.apache.spark.sql.execution.CarbonLateDecodeStrategy
-import org.apache.spark.sql.execution.command.{CreateTable, TableModel, 
TableNewProcessor}
-import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.execution.command.{TableModel, TableNewProcessor}
+import org.apache.spark.sql.hive.{CarbonMetaStore, CarbonRelation}
 import org.apache.spark.sql.optimizer.CarbonLateDecodeRule
 import org.apache.spark.sql.parser.CarbonSpark2SqlParser
 import org.apache.spark.sql.sources._
@@ -34,7 +34,7 @@ import org.apache.spark.sql.types.StructType
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.schema
+import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry
 import org.apache.carbondata.core.metadata.schema.table.TableInfo
 import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
 import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
@@ -130,14 +130,14 @@ class CarbonSource extends CreatableRelationProvider with 
RelationProvider
 if (tableName.contains(" ")) {
   sys.error("Table creation failed. Table name cannot contain blank space")
 }
-val path = if 
(sqlContext.sparkSession.sessionState.catalog.listTables(dbName)
+val (path, updatedParams) = if 
(sqlContext.sparkSession.sessionState.catalog.listTables(dbName)
   .exists(_.table.equalsIgnoreCase(tableName))) {
 getPathForTable(sqlContext.sparkSession, dbName, tableName, parameters)
 } else {
 createTableIfNotExists(sqlContext.sparkSession, parameters, dataSchema)
 }
 
-CarbonDatasourceHadoopRelation(sqlContext.sparkSession, Array(path), 
parameters,
+CarbonDatasourceHadoopRelation(sqlContext.sparkSession, Array(path), 
updatedParams,
   Option(dataSchema))
   }
 
@@ -162,17 +162,14 @@ class CarbonSource extends CreatableRelationProvider with 
RelationProvider
   } else {
 CarbonEnv.getInstance(sparkSession).carbonMetastore
   .lookupRelation(Option(dbName), tableName)(sparkSession)
-CarbonEnv.getInstance(sparkSession).storePath + s"/$dbName/$tableName"
+(CarbonEnv.getInstance(sparkSession).storePath + 
s"/$dbName/$tableName", parameters)
   }
 } catch {
   case ex: NoSuchTableException =>
-val cm: TableModel = CarbonSource.createTableInfoFromParams(
-  parameters,
-  dataSchema,
-  dbName,
-  tableName)
-CreateTable(cm, false).run(sparkSession)
-getPathForTable(sparkSession, dbName, tableName, parameters)
+val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+val updatedParams =
+  CarbonSource.updateAndCreateTable(dataSchema, sparkSession, 
metaStore, parameters)
+getPathForTable(sparkSession, dbName, tableName, updatedParams)
   case ex: Exception =>
 throw new Exception("do not have dbname and tablename for carbon 
table", ex)
 }
@@ -187,7 +184,7 @@ class CarbonSource extends CreatableRelationProvider with 
RelationProvider
* @return
*/
   private def getPathForTable(sparkSession: SparkSession, 

[1/2] carbondata git commit: [CARBONDATA-1450] Support timestamp, int and Long as Dictionary Exclude

[gvramana]

Repository: carbondata
Updated Branches:
  refs/heads/master 8b83f5885 -> 36ceb59f0


http://git-wip-us.apache.org/repos/asf/carbondata/blob/36ceb59f/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SortColumnExcudeDictTestCase.scala
--
diff --git 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SortColumnExcudeDictTestCase.scala
 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SortColumnExcudeDictTestCase.scala
new file mode 100644
index 000..6f9df82
--- /dev/null
+++ 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SortColumnExcudeDictTestCase.scala
@@ -0,0 +1,433 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * Test Class for SortColumnExcudeDictTestCase to verify all scenerios
+ */
+
+class SortColumnExcudeDictTestCase extends QueryTest with BeforeAndAfterAll {
+
+
+  //create table with no dictionary sort_columns
+  test("Sortcolumn-001_TC001", Include) {
+sql(s"""drop table if exists sorttable1""").collect
+sql(s"""CREATE TABLE sorttable1 (empno int, empname String, designation 
String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, 
deptno int, deptname String, projectcode int, projectjoindate Timestamp, 
projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 
'org.apache.carbondata.format' 
tblproperties('dictionary_exclude'='empno','sort_columns'='empno')""").collect
+sql(s"""LOAD DATA local inpath '$resourcesPath/Data/sortcolumns/data.csv' 
INTO TABLE sorttable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""").collect
+sql(s"""select empno from sorttable1""").collect
+
+sql(s"""drop table if exists sorttable1""").collect
+  }
+
+
+  //create table with direct-dictioanry sort_columns
+  test("Sortcolumn-001_TC003", Include) {
+sql(s"""CREATE TABLE sorttable3 (empno int, empname String, designation 
String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, 
deptno int, deptname String, projectcode int, projectjoindate Timestamp, 
projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 
'org.apache.carbondata.format' 
tblproperties('dictionary_include'='doj','sort_columns'='doj')""").collect
+sql(s"""LOAD DATA local inpath '$resourcesPath/Data/sortcolumns/data.csv' 
INTO TABLE sorttable3 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""").collect
+sql(s"""select doj from sorttable3""").collect
+
+sql(s"""drop table if exists sorttable3""").collect
+  }
+
+
+  //create table with multi-sort_columns and data loading with offheap safe
+  test("Sortcolumn-001_TC004", Include) {
+sql(s"""CREATE TABLE sorttable4_offheap_safe (empno int, empname String, 
designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname 
String, deptno int, deptname String, projectcode int, projectjoindate 
Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) 
STORED BY 'org.apache.carbondata.format' 
tblproperties('dictionary_exclude'='workgroupcategory','sort_columns'='workgroupcategory,
 empname')""").collect
+sql(s"""LOAD DATA local inpath '$resourcesPath/Data/sortcolumns/data.csv' 
INTO TABLE sorttable4_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= 
'\"')""").collect
+sql(s"""select workgroupcategory, empname from 
sorttable4_offheap_safe""").collect
+
+sql(s"""drop table if exists sorttable4_offheap_safe""").collect
+  }
+
+
+  //create table with multi-sort_columns and data loading with offheap and 
unsafe sort
+  test("Sortcolumn-001_TC005", Include) {
+sql(s"""CREATE TABLE sorttable4_offheap_unsafe (empno int, empname String, 
designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname 
String, deptno int, deptname 

[2/2] carbondata git commit: [CARBONDATA-1450] Support timestamp, int and Long as Dictionary Exclude

[gvramana]

[CARBONDATA-1450] Support timestamp, int and Long as Dictionary Exclude

Timestamp column supports 68 years.

This PR breaks the limitation of 68 years and can support any time.

To be noted,

(1) By default timestamp will be no dictionary column that can support any 
timestamp without limitation

(2) If it is enough to load only 68 years, then explicitly timestamp column can 
be included in DICTIONARY_INCLUDE(this will be direct_dictionary)

(3) Sort columns support for int,long, bigint

(4) int, long, bigint can be DICTIONARY_EXCLUDE columns

(5) If the timestamp column to be partitioned, it should be a 
DICTIONARY_INCLUDE column.(Partition on timestamp column(dictionary_exclude 
column) will not throw any exception but not supported)

This closes #1322


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/36ceb59f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/36ceb59f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/36ceb59f

Branch: refs/heads/master
Commit: 36ceb59f014f7369575f433064e88aa07a7de48e
Parents: 8b83f58
Author: dhatchayani 
Authored: Tue Sep 5 15:54:28 2017 +0530
Committer: Venkata Ramana G 
Committed: Mon Sep 18 22:34:21 2017 +0530

--
 .../core/constants/CarbonCommonConstants.java   |   6 +
 .../carbondata/core/datastore/TableSpec.java|   4 +
 ...feVariableLengthDimensionDataChunkStore.java |  11 +-
 ...afeVariableLengthDimesionDataChunkStore.java |   7 +-
 .../DictionaryBasedVectorResultCollector.java   |   3 +
 .../RestructureBasedRawResultCollector.java |   5 +-
 .../RestructureBasedVectorResultCollector.java  |  27 +-
 .../executor/impl/AbstractQueryExecutor.java|  18 +-
 .../scan/executor/util/RestructureUtil.java |  21 +-
 .../scan/filter/FilterExpressionProcessor.java  |  24 +-
 .../carbondata/core/scan/filter/FilterUtil.java |  13 +-
 .../executer/RangeValueFilterExecuterImpl.java  |  10 +-
 .../executer/RestructureEvaluatorImpl.java  |   5 +-
 .../executer/RowLevelFilterExecuterImpl.java|   2 +
 ...velRangeLessThanEqualFilterExecuterImpl.java |   2 +
 .../RowLevelRangeLessThanFiterExecuterImpl.java |   2 +
 .../RowLevelRangeFilterResolverImpl.java|  15 +-
 .../carbondata/core/util/DataTypeUtil.java  |  43 +-
 .../sdv/generated/DataLoadingTestCase.scala |  14 +-
 .../sdv/generated/QueriesBVATestCase.scala  |   2 +-
 .../sdv/generated/QueriesBasicTestCase.scala|   2 +-
 .../generated/QueriesCompactionTestCase.scala   |   2 +-
 .../QueriesExcludeDictionaryTestCase.scala  |   2 +-
 .../SortColumnExcudeDictTestCase.scala  | 433 +++
 .../src/test/resources/data_beyond68yrs.csv |  11 +
 .../spark/testsuite/datetype/DateTypeTest.scala |  51 +--
 .../RangeFilterAllDataTypesTestCases.scala  |   1 +
 .../TimestampNoDictionaryColumnTestCase.scala   |  93 
 .../partition/TestDDLForPartitionTable.scala|   2 +-
 ...ForPartitionTableWithDefaultProperties.scala |   5 +-
 .../testsuite/sortcolumns/TestSortColumns.scala |  43 ++
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |   9 +-
 .../command/carbonTableSchemaCommon.scala   |  10 +-
 .../execution/command/carbonTableSchema.scala   |   3 +-
 .../createtable/TestCreateTableSyntax.scala |  15 +-
 .../partition/TestAlterPartitionTable.scala |   6 +-
 .../AlterTableValidationTestCase.scala  |  26 +-
 .../impl/NonDictionaryFieldConverterImpl.java   |  31 +-
 38 files changed, 855 insertions(+), 124 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/36ceb59f/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 3bc1bcc..36d73d7 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -190,6 +190,12 @@ public final class CarbonCommonConstants {
* Bytes for string 0, it is used in codegen in case of null values.
*/
   public static final byte[] ZERO_BYTE_ARRAY = 
"0".getBytes(Charset.forName(DEFAULT_CHARSET));
+
+  /**
+   * Empty byte array
+   */
+  public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
+
   /**
* FILE STATUS IN-PROGRESS
*/

http://git-wip-us.apache.org/repos/asf/carbondata/blob/36ceb59f/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
--
diff --git 

carbondata git commit: [Release]Add dependency of spark-sql-2.1 to presto as it fails when building using spark-1.6 version.

[jackylk]

Repository: carbondata
Updated Branches:
  refs/heads/master cb51b8621 -> 8b83f5885


[Release]Add dependency of spark-sql-2.1 to presto as it fails when building 
using spark-1.6 version.

This closes #1366


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8b83f588
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8b83f588
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8b83f588

Branch: refs/heads/master
Commit: 8b83f5885daabff5eb1b38a146100d5f6123aa5a
Parents: cb51b86
Author: Ravindra Pesala 
Authored: Mon Sep 18 17:06:21 2017 +0530
Committer: Jacky Li 
Committed: Mon Sep 18 21:30:46 2017 +0800

--
 integration/presto/pom.xml | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/8b83f588/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 50e6349..6a225f0 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -438,7 +438,11 @@
 
   
 
-
+
+  org.apache.spark
+  spark-sql_2.11
+  2.1.0
+
 
   org.apache.spark
   spark-network-common_2.11

Jenkins build is still unstable: carbondata-master-spark-1.6 #1883

[Apache Jenkins Server]

See 

Jenkins build is still unstable: carbondata-master-spark-1.6 » Apache CarbonData :: Spark Common Test #1883

[Apache Jenkins Server]

See 

carbondata git commit: [maven-release-plugin] prepare for next development iteration

[ravipesala]

Repository: carbondata
Updated Branches:
  refs/heads/branch-1.2 ede03f5c9 -> 3b4881f1c


[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3b4881f1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3b4881f1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3b4881f1

Branch: refs/heads/branch-1.2
Commit: 3b4881f1cd361edacc6b6df62d8ec86ccb150975
Parents: ede03f5
Author: Ravindra Pesala 
Authored: Mon Sep 18 20:01:18 2017 +0530
Committer: Ravindra Pesala 
Committed: Mon Sep 18 20:01:18 2017 +0530

--
 assembly/pom.xml  | 2 +-
 common/pom.xml| 2 +-
 core/pom.xml  | 2 +-
 examples/flink/pom.xml| 2 +-
 examples/spark/pom.xml| 2 +-
 examples/spark2/pom.xml   | 2 +-
 format/pom.xml| 2 +-
 hadoop/pom.xml| 2 +-
 integration/hive/pom.xml  | 2 +-
 integration/presto/pom.xml| 2 +-
 integration/spark-common-test/pom.xml | 2 +-
 integration/spark-common/pom.xml  | 2 +-
 integration/spark/pom.xml | 2 +-
 integration/spark2/pom.xml| 2 +-
 pom.xml   | 4 ++--
 processing/pom.xml| 2 +-
 16 files changed, 17 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4881f1/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 72a9616..85e59be 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0
+1.2.1-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4881f1/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index b573678..28f3d91 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0
+1.2.1-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4881f1/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index ad60d97..87f9872 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0
+1.2.1-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4881f1/examples/flink/pom.xml
--
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index 4e476fb..1eb0c20 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0
+1.2.1-SNAPSHOT
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4881f1/examples/spark/pom.xml
--
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 18dbc74..0b06d41 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0
+1.2.1-SNAPSHOT
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4881f1/examples/spark2/pom.xml
--
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index a217896..d13fec7 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0
+1.2.1-SNAPSHOT
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4881f1/format/pom.xml
--
diff --git a/format/pom.xml b/format/pom.xml
index a62e444..20f73fd 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0
+1.2.1-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4881f1/hadoop/pom.xml
--
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index c9fde5b..58b1428 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0
+1.2.1-SNAPSHOT
 ../pom.xml
   
 

svn commit: r21650 - in /dev/carbondata/1.2.0-rc2: ./ apache-carbondata-1.2.0-source-release.zip apache-carbondata-1.2.0-source-release.zip.asc apache-carbondata-1.2.0-source-release.zip.md5 apache-ca

[ravipesala]

Author: ravipesala
Date: Mon Sep 18 15:20:30 2017
New Revision: 21650

Log:
Upload 1.2.0-rc2

Added:
dev/carbondata/1.2.0-rc2/
dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip   (with 
props)
dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.asc
dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.md5
dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.sha512

Added: dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip
==
Binary file - no diff available.

Propchange: dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip
--
svn:mime-type = application/octet-stream

Added: dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.asc
==
--- dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.asc 
(added)
+++ dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.asc Mon 
Sep 18 15:20:30 2017
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v1
+
+iQIcBAABAgAGBQJZv9ZdAAoJELrXKninsbLucNcP/2ux2UKJOyB1ZWiYDCF3DeGz
+zc+yh7cy+GsWQixjyXFu5kA++S1v8WHqEjtqsdxk2FGSl8pKnXGojFSce8X0JYH7
+snpx+GADURf4j8+/9TxG4IuT5Tp8AaX+u2mR3Hl6N8Wq8hrq6bfYcvRGvLUOb/a1
+O7BMj78BnWtqTEgxFhdNQpC8BtYs29NeuyzChDfWCJ8S0q/UT+oiR5raFaZ98NYe
+jdO+EtMk7kayZdLfDdesWr7x7KquaDLqQO9Gb6kt64Q5rKDGwrRF37QMW1pvTrPl
+oL2laOXEt8ci5bsNXlT23Y7FbzJ1x6nM3eGx14UaatLFCoPS/zR5twuaZF3DDHpJ
+mlYZTtMWstqc5rPSSJq7sf5xJnKEXZQExiOn72ZUqGoNGB50MZEUJNDPso8SKuRs
+DY76FSzbTGSgX5M48HImVIgVApy9huMixhR2Mou3965M0V2oA+YnOteFUtxGaKKd
+HXhqzROu6ReA4hETB7i4gBwfnH6AHj8FcNyu3SxtZTf7NJlsftl122hYF1rgk3Am
+boqLrc1inSqYYgi7WcgjjAchuVRp5JOm9a6+OwUDl+aryRPMjsVvRUX+zTnYe+1m
+hE0JwYJ1iEeUZxHfeEb2Si/i60HqoS5H2ElC2oyCgJgkOHM4bX9WAL7pyfYF9bSR
+zLv+M4biaTJY7TxOOYvS
+=yfNh
+-END PGP SIGNATURE-

Added: dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.md5
==
--- dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.md5 
(added)
+++ dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.md5 Mon 
Sep 18 15:20:30 2017
@@ -0,0 +1 @@
+ba5b01d28ed5d683eee3e753bfa6315c  apache-carbondata-1.2.0-source-release.zip

Added: 
dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.sha512
==
--- dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.sha512 
(added)
+++ dev/carbondata/1.2.0-rc2/apache-carbondata-1.2.0-source-release.zip.sha512 
Mon Sep 18 15:20:30 2017
@@ -0,0 +1 @@
+b8bd7a56ffbca106d44837ed871c87461405edaea1ba277860d11b542ded01f4c9354c9dc6e8cd8cb8d9d2d6a91c1b2084bc8bcdf7a8cafd70af3f6cf0988d82
  apache-carbondata-1.2.0-source-release.zip

[carbondata] Git Push Summary

[ravipesala]

Repository: carbondata
Updated Tags:  refs/tags/apache-carbondata-1.2.0-rc2 [created] f33d01822

carbondata git commit: [maven-release-plugin] prepare release apache-carbondata-1.2.0-rc2

[ravipesala]

Repository: carbondata
Updated Branches:
  refs/heads/branch-1.2 8b83f5885 -> ede03f5c9


[maven-release-plugin] prepare release apache-carbondata-1.2.0-rc2


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ede03f5c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ede03f5c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ede03f5c

Branch: refs/heads/branch-1.2
Commit: ede03f5c963b13cc640feba799a22466246951c6
Parents: 8b83f58
Author: Ravindra Pesala 
Authored: Mon Sep 18 19:57:56 2017 +0530
Committer: Ravindra Pesala 
Committed: Mon Sep 18 19:57:56 2017 +0530

--
 assembly/pom.xml  |  2 +-
 common/pom.xml|  2 +-
 core/pom.xml  |  2 +-
 examples/flink/pom.xml|  2 +-
 examples/spark/pom.xml|  2 +-
 examples/spark2/pom.xml   |  2 +-
 format/pom.xml|  2 +-
 hadoop/pom.xml|  2 +-
 integration/hive/pom.xml  |  2 +-
 integration/presto/pom.xml|  2 +-
 integration/spark-common-test/pom.xml | 14 +++---
 integration/spark-common/pom.xml  |  2 +-
 integration/spark/pom.xml |  2 +-
 integration/spark2/pom.xml|  2 +-
 pom.xml   |  4 ++--
 processing/pom.xml|  2 +-
 16 files changed, 23 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/ede03f5c/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index f4d4f70..72a9616 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0-SNAPSHOT
+1.2.0
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ede03f5c/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 8b2e002..b573678 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0-SNAPSHOT
+1.2.0
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ede03f5c/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index 1c6d0c4..ad60d97 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0-SNAPSHOT
+1.2.0
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ede03f5c/examples/flink/pom.xml
--
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index d83e2cd..4e476fb 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0-SNAPSHOT
+1.2.0
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ede03f5c/examples/spark/pom.xml
--
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 051237c..18dbc74 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0-SNAPSHOT
+1.2.0
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ede03f5c/examples/spark2/pom.xml
--
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index 1802451..a217896 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0-SNAPSHOT
+1.2.0
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ede03f5c/format/pom.xml
--
diff --git a/format/pom.xml b/format/pom.xml
index e57effd..a62e444 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0-SNAPSHOT
+1.2.0
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ede03f5c/hadoop/pom.xml
--
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index 4df8922..c9fde5b 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.2.0-SNAPSHOT
+1.2.0
 ../pom.xml
   
 

carbondata git commit: [Release]Add dependency of spark-sql-2.1 to presto as it fails when building using spark-1.6 version.

[ravipesala]

Repository: carbondata
Updated Branches:
  refs/heads/branch-1.2 cb51b8621 -> 8b83f5885


[Release]Add dependency of spark-sql-2.1 to presto as it fails when building 
using spark-1.6 version.

This closes #1366


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8b83f588
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8b83f588
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8b83f588

Branch: refs/heads/branch-1.2
Commit: 8b83f5885daabff5eb1b38a146100d5f6123aa5a
Parents: cb51b86
Author: Ravindra Pesala 
Authored: Mon Sep 18 17:06:21 2017 +0530
Committer: Jacky Li 
Committed: Mon Sep 18 21:30:46 2017 +0800

--
 integration/presto/pom.xml | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/8b83f588/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 50e6349..6a225f0 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -438,7 +438,11 @@
 
   
 
-
+
+  org.apache.spark
+  spark-sql_2.11
+  2.1.0
+
 
   org.apache.spark
   spark-network-common_2.11

Jenkins build became unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Spark Common Test #1412

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-2.1 #1412

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-1.6 #1884

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-1.6 » Apache CarbonData :: Spark Common Test #1884

[Apache Jenkins Server]

See 

Build failed in Jenkins: carbondata-master-spark-1.6 #1887

[Apache Jenkins Server]

See 


--
Started by upstream project "carbondata-master-spark-2.1" build number 1414
originally caused by:
 Started by upstream project "carbondata-master-spark-1.6" build number 1886
 originally caused by:
  Started by upstream project "carbondata-master-spark-2.1" build number 1413
  originally caused by:
   Started by an SCM change
[EnvInject] - Loading node environment variables.
Building remotely on H14 (couchdbtest ubuntu xenial) in workspace 

 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url 
 > https://git-wip-us.apache.org/repos/asf/carbondata.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from 
https://git-wip-us.apache.org/repos/asf/carbondata.git
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:817)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1084)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1115)
at hudson.scm.SCM.checkout(SCM.java:495)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1276)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:560)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:485)
at hudson.model.Run.execute(Run.java:1735)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:542)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:405)
Caused by: hudson.plugins.git.GitException: Command "git config 
remote.origin.url https://git-wip-us.apache.org/repos/asf/carbondata.git; 
returned status code 4:
stdout: 
stderr: error: failed to write new configuration file 


at 
org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1924)
at 
org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1892)
at 
org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1888)
at 
org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1533)
at 
org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1545)
at 
org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1179)
at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
at sun.reflect.GeneratedMethodAccessor45.invoke(Unknown Source)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:895)
at 
hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:870)
at 
hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:829)
at hudson.remoting.UserRequest.perform(UserRequest.java:153)
at hudson.remoting.UserRequest.perform(UserRequest.java:50)
at hudson.remoting.Request$2.run(Request.java:336)
at 
hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:68)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
at ..remote call to H14(Native Method)
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1545)
at hudson.remoting.UserResponse.retrieve(UserRequest.java:253)
at hudson.remoting.Channel.call(Channel.java:830)
at 
hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:257)
at com.sun.proxy.$Proxy104.setRemoteUrl(Unknown Source)
at 
org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:805)
... 11 more
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url 
 > https://git-wip-us.apache.org/repos/asf/carbondata.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from 
https://git-wip-us.apache.org/repos/asf/carbondata.git
at 

Jenkins build is back to stable : carbondata-master-spark-2.1 » Apache CarbonData :: Spark Common Test #1413

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.1 #1413

[Apache Jenkins Server]

See 

carbondata git commit: [CARBONDATA-1425] Inappropriate Exception displays while creating a new partition with incorrect partition type

[chenliang613]

Repository: carbondata
Updated Branches:
  refs/heads/master d3cb6f653 -> 7e5e29c74


[CARBONDATA-1425] Inappropriate Exception displays while creating a new 
partition with incorrect partition type

Change the error content when the range info data mismatch the partition 
field's data type, the new showing content as below:Data in range info must be 
the same type with the partition field's type

This closes #1336


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7e5e29c7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7e5e29c7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7e5e29c7

Branch: refs/heads/master
Commit: 7e5e29c740e1b1a253379ed969b260accc73b09f
Parents: d3cb6f6
Author: mayun 
Authored: Wed Sep 6 22:52:39 2017 +0800
Committer: chenliang613 
Committed: Mon Sep 18 14:48:25 2017 +0800

--
 .../carbondata/spark/util/CommonUtil.scala  |   7 +-
 .../partition/TestAlterPartitionTable.scala | 118 ++-
 2 files changed, 123 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/7e5e29c7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index ed4d784..fd265a8 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -334,12 +334,17 @@ object CommonUtil {
 timestampFormatter, dateFormatter)
 }
 val iterator = rangeInfo.tail.toIterator
-while(iterator.hasNext) {
+while (iterator.hasNext) {
   val next = columnDataType match {
 case DataType.STRING => ByteUtil.toBytes(iterator.next())
 case _ => PartitionUtil.getDataBasedOnDataType(iterator.next(), 
columnDataType,
   timestampFormatter, dateFormatter)
   }
+  if (next == null) {
+sys.error(
+  "Data in range info must be the same type with the partition field's 
type "
++ columnDataType)
+  }
   if (comparator.compare(head, next) < 0) {
 head = next
   } else {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/7e5e29c7/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index 9de2ef5..0c59bd9 100644
--- 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -36,7 +36,8 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 
   override def beforeAll {
 dropTable
-
+CarbonProperties.getInstance()
+  .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "-MM-dd")
 CarbonProperties.getInstance()
   .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "/MM/dd")
 /**
@@ -569,6 +570,111 @@ class TestAlterPartitionTable extends QueryTest with 
BeforeAndAfterAll {
 checkAnswer(result_after5, result_origin5)
   }
 
+   test("test exception when alter partition and the values"
+   + "in range_info can not match partition column type") {
+ val exception_test_range_int: Exception = intercept[Exception] {
+  sql(
+"""
+  | CREATE TABLE test_range_int(col1 INT, col2 STRING)
+  | PARTITIONED BY (col3 INT) STORED BY 'carbondata'
+  | TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='11,12')
+""".stripMargin)
+   sql("ALTER TABLE test_range_int ADD PARTITION ('abc')")
+}
+ assert(exception_test_range_int.getMessage
+ .contains("Data in range info must be the same type with the 
partition field's type"))
+
+sql("DROP TABLE IF EXISTS test_range_smallint")
+val exception_test_range_smallint: Exception = intercept[Exception] {
+  sql(
+"""
+  | CREATE TABLE test_range_smallint(col1 INT, col2 STRING)
+  | PARTITIONED BY (col3 SMALLINT) STORED BY 'carbondata'
+  | TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='11,12')
+

Jenkins build is back to stable : carbondata-master-spark-1.6 » Apache CarbonData :: Spark Common Test #1875

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-1.6 #1875

[Apache Jenkins Server]

See 

carbondata git commit: [CARBONDATA-1477] Corrected the Bug For Wrong Values Of Date data type in hive

[chenliang613]

Repository: carbondata
Updated Branches:
  refs/heads/master 7e5e29c74 -> c2574688f


[CARBONDATA-1477] Corrected the Bug For Wrong Values Of Date data type in hive

This closes #1356


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c2574688
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c2574688
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c2574688

Branch: refs/heads/master
Commit: c2574688f82d4bd9bfea6e33d04d2ae1f96ea0f1
Parents: 7e5e29c
Author: anubhav100 
Authored: Wed Sep 13 17:50:19 2017 +0530
Committer: chenliang613 
Committed: Mon Sep 18 16:06:36 2017 +0800

--
 .../carbondata/hive/CarbonDictionaryDecodeReadSupport.java  | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/c2574688/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
--
diff --git 
a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
 
b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
index 76597fd..34a1936 100644
--- 
a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
+++ 
b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.sql.Date;
 import java.sql.Timestamp;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.List;
 
 import org.apache.carbondata.core.cache.Cache;
@@ -228,9 +229,13 @@ public class CarbonDictionaryDecodeReadSupport 
implements CarbonReadSupport