date:20180411

carbondata git commit: [CARBONDATA-2301][SDK] CarbonStore interface and two implementations (Spark and Local)

2018-04-11 Thread ravipesala

Repository: carbondata
Updated Branches:
  refs/heads/master 3740535d5 -> cfb8ed9f5


[CARBONDATA-2301][SDK] CarbonStore interface and two implementations (Spark and 
Local)

User should be able to query carbondata using CarbonStore interface.

Get API: It can be used for filter query. It accepts projection column names 
and filter expression, and returns matched rows.
SQL API: it accepts SQL statement and return query result set.

This closes #2127


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/cfb8ed9f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/cfb8ed9f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/cfb8ed9f

Branch: refs/heads/master
Commit: cfb8ed9f5ec932911ddefe8fc2dcd07269411396
Parents: 3740535
Author: Jacky Li 
Authored: Sun Apr 1 16:30:22 2018 +0800
Committer: ravipesala 
Committed: Thu Apr 12 07:20:40 2018 +0530

--
 .../carbondata/core/util/DataTypeUtil.java  |   8 +-
 .../hadoop/api/CarbonInputFormat.java   |  23 ++--
 .../dataload/TestGlobalSortDataLoad.scala   |   5 +-
 .../StandardPartitionTableLoadingTestCase.scala |   6 +-
 .../StandardPartitionTableQueryTestCase.scala   |   5 +-
 .../carbondata/spark/rdd/CarbonScanRDD.scala|  25 ++--
 .../apache/spark/sql/test/util/QueryTest.scala  |   1 +
 integration/spark2/pom.xml  |   5 +
 .../carbondata/store/SparkCarbonStore.scala |  98 ++
 .../sql/CarbonDatasourceHadoopRelation.scala|   2 +-
 .../spark/sql/CarbonDictionaryDecoder.scala |  39 +-
 .../org/apache/spark/sql/CarbonSession.scala|   2 +-
 .../strategy/CarbonLateDecodeStrategy.scala |   4 +-
 .../apache/spark/sql/hive/CarbonRelation.scala  |  69 ++
 .../apache/spark/util/SparkTypeConverter.scala  | 135 +++
 .../carbondata/store/SparkCarbonStoreTest.scala |  86 
 store/sdk/pom.xml   |   5 +
 .../carbondata/store/CarbonRowReadSupport.java  |  53 
 .../apache/carbondata/store/CarbonStore.java|  68 ++
 .../carbondata/store/LocalCarbonStore.java  | 116 
 .../carbondata/store/MetaCachedCarbonStore.java |  54 
 .../apache/carbondata/sdk/file/TestUtil.java|   4 +-
 .../carbondata/store/LocalCarbonStoreTest.java  |  61 +
 23 files changed, 743 insertions(+), 131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/cfb8ed9f/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java 
b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 25f0099..6967102 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -801,9 +801,11 @@ public final class DataTypeUtil {
* @param converterLocal
*/
   public static void setDataTypeConverter(DataTypeConverter converterLocal) {
-converter = converterLocal;
-timeStampformatter.remove();
-dateformatter.remove();
+if (converterLocal != null) {
+  converter = converterLocal;
+  timeStampformatter.remove();
+  dateformatter.remove();
+}
   }
 
   public static DataTypeConverter getDataTypeConverter() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/cfb8ed9f/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
index be97d05..b1d7603 100644
--- 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
+++ 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
@@ -513,24 +513,27 @@ public abstract class CarbonInputFormat extends 
FileInputFormat {
* It is optional, if user does not set then it reads from store
*
* @param configuration
-   * @param converter is the Data type converter for different computing engine
-   * @throws IOException
+   * @param converterClass is the Data type converter for different computing 
engine
*/
-  public static void setDataTypeConverter(Configuration configuration, 
DataTypeConverter converter)
-  throws IOException {
-if (null != converter) {
-  configuration.set(CARBON_CONVERTER,
-  ObjectSerializationUtil.convertObjectToString(converter));
+  public static void setDataTypeConverter(
+  Configuration configuration, Class 
converterClass) {
+

Jenkins build is back to stable : carbondata-master-spark-2.1 #2247

2018-04-11 Thread Apache Jenkins Server

See

Jenkins build is back to stable : carbondata-master-spark-2.1 » Apache CarbonData :: Spark2 #2247

2018-04-11 Thread Apache Jenkins Server

See

Jenkins build is unstable: carbondata-master-spark-2.1 #2246

2018-04-11 Thread Apache Jenkins Server

See

Jenkins build is back to normal : carbondata-master-spark-2.1 » Apache CarbonData :: Store SDK #2246

2018-04-11 Thread Apache Jenkins Server

See

Jenkins build became unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Spark2 #2246

2018-04-11 Thread Apache Jenkins Server

See

carbondata git commit: [CARBONDATA-2267] [Presto] Support Reading CarbonData Standard Hive Partition From Presto Integration

2018-04-11 Thread chenliang613

Repository: carbondata
Updated Branches:
  refs/heads/master f1d84646a -> 3740535d5


[CARBONDATA-2267] [Presto] Support Reading CarbonData Standard Hive Partition 
From Presto Integration

Support Reading CarbonData Standard Hive Partition From Presto Integration

This closes #2139


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3740535d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3740535d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3740535d

Branch: refs/heads/master
Commit: 3740535d50e787df547dec2d2998591dfd1f9875
Parents: f1d8464
Author: anubhav100 
Authored: Wed Apr 4 13:48:15 2018 +0530
Committer: chenliang613 
Committed: Wed Apr 11 23:10:32 2018 +0800

--
 .../presto/CarbondataSplitManager.java  |   2 +-
 .../carbondata/presto/PrestoFilterUtil.java |  84 ++-
 .../presto/impl/CarbonTableReader.java  | 108 +++
 3 files changed, 171 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/3740535d/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
--
diff --git 
a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
 
b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
index db2a5e7..811393f 100755
--- 
a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
+++ 
b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
@@ -69,7 +69,7 @@ public class CarbondataSplitManager implements 
ConnectorSplitManager {
 CarbonTableCacheModel cache = carbonTableReader.getCarbonCache(key);
 Expression filters = 
PrestoFilterUtil.parseFilterExpression(layoutHandle.getConstraint());
 try {
-  List splits = 
carbonTableReader.getInputSplits2(cache, filters);
+  List splits = 
carbonTableReader.getInputSplits2(cache, filters, layoutHandle.getConstraint());
 
   ImmutableList.Builder cSplits = ImmutableList.builder();
   for (CarbonLocalInputSplit split : splits) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/3740535d/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
--
diff --git 
a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
 
b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
index 6f9b0f3..65f5eb1 100644
--- 
a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
+++ 
b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
@@ -19,16 +19,18 @@ package org.apache.carbondata.presto;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.scan.expression.ColumnExpression;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.LiteralExpression;
@@ -43,6 +45,7 @@ import 
org.apache.carbondata.core.scan.expression.logical.AndExpression;
 import org.apache.carbondata.core.scan.expression.logical.OrExpression;
 
 import com.facebook.presto.spi.ColumnHandle;
+import com.facebook.presto.spi.PrestoException;
 import com.facebook.presto.spi.predicate.Domain;
 import com.facebook.presto.spi.predicate.Range;
 import com.facebook.presto.spi.predicate.TupleDomain;
@@ -60,7 +63,10 @@ import com.facebook.presto.spi.type.VarcharType;
 import com.google.common.collect.ImmutableList;
 import io.airlift.slice.Slice;
 
+import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
 import static com.google.common.base.Preconditions.checkArgument;
+import static java.lang.String.format;
+import static java.util.stream.Collectors.toList;
 
 /**
  * PrestoFilterUtil create the carbonData Expression from the presto-domain
@@ -69,6 +75,12 @@ public class PrestoFilterUtil {
 
   private static Map filterMap = new HashMap<>();
 
+  private final static String HIVE_DEFAULT_DYNAMIC_PARTITION = 
"__HIVE_DEFAULT_PARTITION__";
+
+  /**

Jenkins build is back to stable : carbondata-master-spark-2.2 #281

2018-04-11 Thread Apache Jenkins Server

See

Jenkins build is back to stable : carbondata-master-spark-2.2 » Apache CarbonData :: Spark Common Test #281

2018-04-11 Thread Apache Jenkins Server

See

Build failed in Jenkins: carbondata-master-spark-2.1 #2245

2018-04-11 Thread Apache Jenkins Server

See 


--
Started by an SCM change
[EnvInject] - Loading node environment variables.
Building remotely on H24 (ubuntu xenial) in workspace 

java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Retrying after 10 seconds
java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Retrying after 10 seconds
java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Not sending mail to unregistered user ramana.gollam...@huawei.com

carbondata git commit: [CARBONDATA-2303] clean files issue resolved for partition folder

2018-04-11 Thread manishgupta88

Repository: carbondata
Updated Branches:
  refs/heads/branch-1.3 ea751bc7a -> 167260da8


[CARBONDATA-2303] clean files issue resolved for partition folder

This closes #2159


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/167260da
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/167260da
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/167260da

Branch: refs/heads/branch-1.3
Commit: 167260da8786757dff5b5c53caebf1d1bced50fb
Parents: ea751bc
Author: rahulforallp 
Authored: Sun Apr 1 17:38:51 2018 +0530
Committer: manishgupta88 
Committed: Wed Apr 11 18:34:14 2018 +0530

--
 .../filesystem/AbstractDFSCarbonFile.java   | 15 
 .../datastore/filesystem/AlluxioCarbonFile.java | 12 
 .../core/datastore/filesystem/CarbonFile.java   |  3 +
 .../datastore/filesystem/HDFSCarbonFile.java| 13 
 .../datastore/filesystem/LocalCarbonFile.java   | 20 ++
 .../datastore/filesystem/ViewFSCarbonFile.java  | 13 
 .../core/util/path/CarbonTablePath.java | 17 +
 .../org/apache/carbondata/api/CarbonStore.scala | 73 +++-
 8 files changed, 164 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/167260da/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index a4a92ce..af6365b 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -492,6 +492,18 @@ public abstract  class AbstractDFSCarbonFile implements 
CarbonFile {
   }
 
   @Override
+  public List listFiles(Boolean recursive) throws IOException {
+RemoteIterator listStatus = null;
+if (null != fileStatus && fileStatus.isDirectory()) {
+  Path path = fileStatus.getPath();
+  listStatus = 
path.getFileSystem(FileFactory.getConfiguration()).listFiles(path, recursive);
+} else {
+  return new ArrayList();
+}
+return getFiles(listStatus);
+  }
+
+  @Override
   public CarbonFile[] locationAwareListFiles() throws IOException {
 if (null != fileStatus && fileStatus.isDirectory()) {
   List listStatus = new ArrayList<>();
@@ -511,6 +523,9 @@ public abstract  class AbstractDFSCarbonFile implements 
CarbonFile {
*/
   protected abstract CarbonFile[] getFiles(FileStatus[] listStatus);
 
+  protected abstract List 
getFiles(RemoteIterator listStatus)
+  throws IOException;
+
   @Override public String[] getLocations() throws IOException {
 BlockLocation[] blkLocations;
 if (fileStatus instanceof LocatedFileStatus) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/167260da/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
index e0df6ae..61316f8 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
@@ -27,7 +27,9 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
 
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 
 
@@ -67,6 +69,16 @@ public class AlluxioCarbonFile extends AbstractDFSCarbonFile 
{
 return files;
   }
 
+  @Override
+  protected List getFiles(RemoteIterator 
listStatus)
+  throws IOException {
+List carbonFiles = new ArrayList<>();
+while (listStatus.hasNext()) {
+  Path filePath = listStatus.next().getPath();
+  carbonFiles.add(new AlluxioCarbonFile(filePath));
+}
+return carbonFiles;
+  }
 
   @Override
   public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/167260da/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java
--
diff --git

Build failed in Jenkins: carbondata-master-spark-2.1 #2244

2018-04-11 Thread Apache Jenkins Server

See 


--
Started by an SCM change
[EnvInject] - Loading node environment variables.
Building remotely on H24 (ubuntu xenial) in workspace 

java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Retrying after 10 seconds
java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Retrying after 10 seconds
java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Not sending mail to unregistered user ramana.gollam...@huawei.com

Build failed in Jenkins: carbondata-master-spark-2.1 #2243

2018-04-11 Thread Apache Jenkins Server

See 


--
Started by an SCM change
[EnvInject] - Loading node environment variables.
Building remotely on H24 (ubuntu xenial) in workspace 

java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Retrying after 10 seconds
java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Retrying after 10 seconds
java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Not sending mail to unregistered user ramana.gollam...@huawei.com

carbondata git commit: [CARBONDATA-2316] Executor task is failed but UI shows success issue fixed

2018-04-11 Thread manishgupta88

Repository: carbondata
Updated Branches:
  refs/heads/branch-1.3 28f33746a -> ea751bc7a


[CARBONDATA-2316] Executor task is failed but UI shows success issue fixed

Executor task is failed but UI shows success issue for comapction

This closes #2158


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ea751bc7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ea751bc7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ea751bc7

Branch: refs/heads/branch-1.3
Commit: ea751bc7aae477f2ee0417ce61968a22509d660a
Parents: 28f3374
Author: rahulforallp 
Authored: Fri Apr 6 14:16:06 2018 +0530
Committer: manishgupta88 
Committed: Wed Apr 11 18:21:51 2018 +0530

--
 .../core/indexstore/blockletindex/BlockletDataMapFactory.java | 2 ++
 .../org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala | 2 +-
 .../carbondata/processing/merger/AbstractResultProcessor.java | 2 +-
 .../processing/merger/CompactionResultSortProcessor.java  | 6 +++---
 .../processing/merger/RowResultMergerProcessor.java   | 7 +++
 5 files changed, 10 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/ea751bc7/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
 
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index c08c87e..6d8feeb 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -103,6 +103,8 @@ public class BlockletDataMapFactory implements 
DataMapFactory, BlockletDetailsFe
   Set validIdentifiers = 
BlockletDataMapUtil
   
.filterIdentifiersBasedOnDistributable(tableBlockIndexUniqueIdentifiers,
   (BlockletDataMapDistributable) distributable);
+  ((BlockletDataMapDistributable) distributable)
+  .setTableBlockIndexUniqueIdentifiers(validIdentifiers);
   for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : 
validIdentifiers) {
 if (null == cache.getIfPresent(tableBlockIndexUniqueIdentifier)) {
   distributablesToBeLoaded.add(distributable);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ea751bc7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 6932c79..31f3c85 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -230,7 +230,7 @@ class CarbonMergerRDD[K, V](
 
   } catch {
 case e: Exception =>
-  LOGGER.error(e)
+  LOGGER.error(e, "Compaction Failed ")
   throw e
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ea751bc7/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
 
b/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
index 067a4c4..59f2e0d 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
@@ -39,7 +39,7 @@ public abstract class AbstractResultProcessor {
* @param resultIteratorList
* @return
*/
-  public abstract boolean execute(List resultIteratorList);
+  public abstract boolean execute(List resultIteratorList) 
throws Exception;
 
   /**
* This method will be sued to clean up the resources and close all the 
spawned threads to avoid

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ea751bc7/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java

Build failed in Jenkins: carbondata-master-spark-2.1 #2242

2018-04-11 Thread Apache Jenkins Server

See 


--
Started by an SCM change
[EnvInject] - Loading node environment variables.
Building remotely on H24 (ubuntu xenial) in workspace 

java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Retrying after 10 seconds
java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Retrying after 10 seconds
java.io.IOException: Failed to mkdirs: 

at hudson.FilePath.mkdirs(FilePath.java:1170)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1200)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Not sending mail to unregistered user ramana.gollam...@huawei.com

carbondata git commit: [CARBONDATA-2316] Executor task is failed but UI shows success issue is fixed

2018-04-11 Thread manishgupta88

Repository: carbondata
Updated Branches:
  refs/heads/master 13cdeb9f4 -> f1d84646a


[CARBONDATA-2316] Executor task is failed but UI shows success issue is fixed

Executor task is failed but UI shows success issue is fixed for compaction

This closes #2142


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f1d84646
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f1d84646
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f1d84646

Branch: refs/heads/master
Commit: f1d84646a50ab6e47310419947d5045c767f2b75
Parents: 13cdeb9
Author: rahulforallp 
Authored: Fri Apr 6 14:16:06 2018 +0530
Committer: manishgupta88 
Committed: Wed Apr 11 18:19:01 2018 +0530

--
 .../org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala | 2 +-
 .../carbondata/processing/merger/AbstractResultProcessor.java | 2 +-
 .../processing/merger/CompactionResultSortProcessor.java  | 6 +++---
 .../processing/merger/RowResultMergerProcessor.java   | 7 +++
 4 files changed, 8 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1d84646/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index b4c5e4d..6cf8cb3 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -225,7 +225,7 @@ class CarbonMergerRDD[K, V](
 
   } catch {
 case e: Exception =>
-  LOGGER.error(e)
+  LOGGER.error(e, "Compaction Failed ")
   throw e
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1d84646/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
 
b/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
index 3f46a24..7a11c8b 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
@@ -37,7 +37,7 @@ public abstract class AbstractResultProcessor {
* @param resultIteratorList
* @return
*/
-  public abstract boolean execute(List resultIteratorList);
+  public abstract boolean execute(List resultIteratorList) 
throws Exception;
 
   /**
* This method will be sued to clean up the resources and close all the 
spawned threads to avoid

http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1d84646/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
 
b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 850ceca..dd8f739 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -150,7 +150,7 @@ public class CompactionResultSortProcessor extends 
AbstractResultProcessor {
*
* @param resultIteratorList
*/
-  public boolean execute(List resultIteratorList) {
+  public boolean execute(List resultIteratorList) throws 
Exception {
 boolean isCompactionSuccess = false;
 try {
   initTempStoreLocation();
@@ -166,7 +166,7 @@ public class CompactionResultSortProcessor extends 
AbstractResultProcessor {
   }
   isCompactionSuccess = true;
 } catch (Exception e) {
-  LOGGER.error(e, "Compaction failed: " + e.getMessage());
+  throw e;
 } finally {
   if (partitionSpec != null) {
 try {
@@ -175,8 +175,8 @@ public class CompactionResultSortProcessor extends 
AbstractResultProcessor {
   partitionSpec.getLocation().toString(), 
carbonLoadModel.getFactTimeStamp() + "",
   partitionSpec.getPartitions());
 } catch (IOException e) {
-  LOGGER.error(e, "Compaction failed: " + e.getMessage());
   isCompactionSuccess = false;
+  throw e;
 }

Build failed in Jenkins: carbondata-master-spark-2.1 #2241

2018-04-11 Thread Apache Jenkins Server

See 


Changes:

[ramana.gollamudi] [CARBONDATA-2303] clean files issue resolved for partition 
folder

--
[...truncated 1.63 MB...]
[INFO] Including org.apache.hadoop:hadoop-mapreduce-client-core:jar:2.7.2 in 
the shaded jar.
[INFO] Including org.apache.hadoop:hadoop-yarn-common:jar:2.7.2 in the shaded 
jar.
[INFO] Including com.sun.jersey:jersey-client:jar:1.9 in the shaded jar.
[INFO] Including org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:2.7.2 
in the shaded jar.
[INFO] Including org.apache.zookeeper:zookeeper:jar:3.4.7 in the shaded jar.
[INFO] Including org.roaringbitmap:RoaringBitmap:jar:0.5.11 in the shaded jar.
[INFO] Including io.netty:netty-all:jar:4.0.42.Final in the shaded jar.
[INFO] Including net.jpountz.lz4:lz4:jar:1.3.0 in the shaded jar.
[INFO] Including org.scala-lang:scala-library:jar:2.11.8 in the shaded jar.
[INFO] Including com.univocity:univocity-parsers:jar:2.2.1 in the shaded jar.
[INFO] Including org.apache.commons:commons-lang3:jar:3.3.2 in the shaded jar.
[INFO] Including org.apache.hadoop:hadoop-common:jar:2.7.2 in the shaded jar.
[INFO] Including org.apache.hadoop:hadoop-annotations:jar:2.7.2 in the shaded 
jar.
[INFO] Including com.google.guava:guava:jar:11.0.2 in the shaded jar.
[INFO] Including commons-cli:commons-cli:jar:1.2 in the shaded jar.
[INFO] Including org.apache.commons:commons-math3:jar:3.1.1 in the shaded jar.
[INFO] Including xmlenc:xmlenc:jar:0.52 in the shaded jar.
[INFO] Including commons-httpclient:commons-httpclient:jar:3.1 in the shaded 
jar.
[INFO] Including commons-codec:commons-codec:jar:1.4 in the shaded jar.
[INFO] Including commons-io:commons-io:jar:2.4 in the shaded jar.
[INFO] Including commons-net:commons-net:jar:3.1 in the shaded jar.
[INFO] Including commons-collections:commons-collections:jar:3.2.2 in the 
shaded jar.
[INFO] Including org.mortbay.jetty:jetty:jar:6.1.26 in the shaded jar.
[INFO] Including org.mortbay.jetty:jetty-util:jar:6.1.26 in the shaded jar.
[INFO] Including com.sun.jersey:jersey-core:jar:1.9 in the shaded jar.
[INFO] Including com.sun.jersey:jersey-json:jar:1.9 in the shaded jar.
[INFO] Including org.codehaus.jettison:jettison:jar:1.1 in the shaded jar.
[INFO] Including com.sun.xml.bind:jaxb-impl:jar:2.2.3-1 in the shaded jar.
[INFO] Including javax.xml.bind:jaxb-api:jar:2.2.2 in the shaded jar.
[INFO] Including javax.xml.stream:stax-api:jar:1.0-2 in the shaded jar.
[INFO] Including javax.activation:activation:jar:1.1 in the shaded jar.
[INFO] Including org.codehaus.jackson:jackson-jaxrs:jar:1.8.3 in the shaded jar.
[INFO] Including org.codehaus.jackson:jackson-xc:jar:1.8.3 in the shaded jar.
[INFO] Including com.sun.jersey:jersey-server:jar:1.9 in the shaded jar.
[INFO] Including asm:asm:jar:3.1 in the shaded jar.
[INFO] Including commons-logging:commons-logging:jar:1.1.3 in the shaded jar.
[INFO] Including log4j:log4j:jar:1.2.17 in the shaded jar.
[INFO] Including net.java.dev.jets3t:jets3t:jar:0.9.0 in the shaded jar.
[INFO] Including org.apache.httpcomponents:httpclient:jar:4.1.2 in the shaded 
jar.
[INFO] Including org.apache.httpcomponents:httpcore:jar:4.1.2 in the shaded jar.
[INFO] Including com.jamesmurty.utils:java-xmlbuilder:jar:0.4 in the shaded jar.
[INFO] Including commons-lang:commons-lang:jar:2.6 in the shaded jar.
[INFO] Including commons-configuration:commons-configuration:jar:1.6 in the 
shaded jar.
[INFO] Including commons-digester:commons-digester:jar:1.8 in the shaded jar.
[INFO] Including commons-beanutils:commons-beanutils:jar:1.7.0 in the shaded 
jar.
[INFO] Including commons-beanutils:commons-beanutils-core:jar:1.8.0 in the 
shaded jar.
[INFO] Including org.slf4j:slf4j-log4j12:jar:1.7.10 in the shaded jar.
[INFO] Including com.google.protobuf:protobuf-java:jar:2.5.0 in the shaded jar.
[INFO] Including org.apache.hadoop:hadoop-auth:jar:2.7.2 in the shaded jar.
[INFO] Including 
org.apache.directory.server:apacheds-kerberos-codec:jar:2.0.0-M15 in the shaded 
jar.
[INFO] Including org.apache.directory.server:apacheds-i18n:jar:2.0.0-M15 in the 
shaded jar.
[INFO] Including org.apache.directory.api:api-asn1-api:jar:1.0.0-M20 in the 
shaded jar.
[INFO] Including org.apache.directory.api:api-util:jar:1.0.0-M20 in the shaded 
jar.
[INFO] Including org.apache.curator:curator-framework:jar:2.7.1 in the shaded 
jar.
[INFO] Including com.jcraft:jsch:jar:0.1.42 in the shaded jar.
[INFO] Including org.apache.curator:curator-client:jar:2.7.1 in the shaded jar.
[INFO] Including org.apache.curator:curator-recipes:jar:2.7.1 in the shaded jar.
[INFO] Including com.google.code.findbugs:jsr305:jar:3.0.0 in the shaded jar.
[INFO] Including org.apache.htrace:htrace-core:jar:3.1.0-incubating in the 
shaded jar.
[INFO] Including org.apache.hadoop:hadoop-hdfs:jar:2.7.2 in the shaded jar.
[INFO] Including commons-daemon:commons-daemon:jar:1.0.13 in the shaded jar.
[INFO]

Build failed in Jenkins: carbondata-master-spark-2.1 » Apache CarbonData :: Store SDK #2241

2018-04-11 Thread Apache Jenkins Server

See 


--
[INFO] 
[INFO] 
[INFO] Building Apache CarbonData :: Store SDK 1.4.0-SNAPSHOT
[INFO] 
[INFO] 
[INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ carbondata-store-sdk 
---
[INFO] Deleting 

[INFO] 
[INFO] --- jacoco-maven-plugin:0.7.9:prepare-agent (default-prepare-agent) @ 
carbondata-store-sdk ---
[INFO] argLine set to 
-javaagent:/home/jenkins/jenkins-slave/maven-repositories/0/org/jacoco/org.jacoco.agent/0.7.9/org.jacoco.agent-0.7.9-runtime.jar=destfile=
[INFO] 
[INFO] --- maven-remote-resources-plugin:1.5:process (default) @ 
carbondata-store-sdk ---
[INFO] 
[INFO] --- maven-resources-plugin:2.7:resources (default-resources) @ 
carbondata-store-sdk ---
[INFO] Using 'UTF-8' encoding to copy filtered resources.
[INFO] Copying 19 resources
[INFO] Copying 3 resources
[INFO] 
[INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ 
carbondata-store-sdk ---
[INFO] Changes detected - recompiling the module!
[INFO] Compiling 8 source files to 

[INFO] 
:
 

 uses or overrides a deprecated API.
[INFO] 
:
 Recompile with -Xlint:deprecation for details.
[INFO] 
:
 

 uses unchecked or unsafe operations.
[INFO] 
:
 Recompile with -Xlint:unchecked for details.
[INFO] 
[INFO] >>> findbugs-maven-plugin:3.0.4:check (analyze-compile) > :findbugs @ 
carbondata-store-sdk >>>
[INFO] 
[INFO] --- findbugs-maven-plugin:3.0.4:findbugs (findbugs) @ 
carbondata-store-sdk ---
[INFO] Fork Value is true
[INFO] Done FindBugs Analysis
[INFO] 
[INFO] <<< findbugs-maven-plugin:3.0.4:check (analyze-compile) < :findbugs @ 
carbondata-store-sdk <<<
[INFO] 
[INFO] --- findbugs-maven-plugin:3.0.4:check (analyze-compile) @ 
carbondata-store-sdk ---
[INFO] BugInstance size is 0
[INFO] Error size is 0
[INFO] No errors/warnings found
[INFO] 
[INFO] --- maven-resources-plugin:2.7:testResources (default-testResources) @ 
carbondata-store-sdk ---
[INFO] Using 'UTF-8' encoding to copy filtered resources.
[INFO] skip non existing resourceDirectory 

[INFO] Copying 3 resources
[INFO] 
[INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ 
carbondata-store-sdk ---
[INFO] Changes detected - recompiling the module!
[INFO] Compiling 5 source files to 

[INFO] 
[INFO] --- maven-surefire-plugin:2.18.1:test (default-test) @ 
carbondata-store-sdk ---
[INFO] Surefire report directory: 


---
 T E S T S
---
Running org.apache.carbondata.sdk.file.CarbonReaderTest
log4j:WARN No appenders could be found for logger 
(org.apache.carbondata.core.metadata.schema.table.TableInfo).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more 
info.
Tests run: 3, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 3.147 sec -

carbondata git commit: [CARBONDATA-2303] clean files issue resolved for partition folder

2018-04-11 Thread gvramana

Repository: carbondata
Updated Branches:
  refs/heads/master 359f6e6b2 -> 13cdeb9f4


[CARBONDATA-2303] clean files issue resolved for partition folder

This closes #2128


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/13cdeb9f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/13cdeb9f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/13cdeb9f

Branch: refs/heads/master
Commit: 13cdeb9f4f9e3252a9fe4e419c1e7b10e827e390
Parents: 359f6e6
Author: rahulforallp 
Authored: Sun Apr 1 17:38:51 2018 +0530
Committer: Venkata Ramana G 
Committed: Wed Apr 11 17:33:45 2018 +0530

--
 .../filesystem/AbstractDFSCarbonFile.java   | 15 
 .../datastore/filesystem/AlluxioCarbonFile.java | 12 
 .../core/datastore/filesystem/CarbonFile.java   |  3 +
 .../datastore/filesystem/HDFSCarbonFile.java| 13 
 .../datastore/filesystem/LocalCarbonFile.java   | 20 ++
 .../datastore/filesystem/ViewFSCarbonFile.java  | 13 
 .../core/util/path/CarbonTablePath.java | 18 +
 .../org/apache/carbondata/api/CarbonStore.scala | 72 +++-
 8 files changed, 165 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/13cdeb9f/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index bf3292b..03e3de4 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -514,6 +514,18 @@ public abstract class AbstractDFSCarbonFile implements 
CarbonFile {
   }
 
   @Override
+  public List listFiles(Boolean recursive) throws IOException {
+RemoteIterator listStatus = null;
+if (null != fileStatus && fileStatus.isDirectory()) {
+  Path path = fileStatus.getPath();
+  listStatus = 
path.getFileSystem(FileFactory.getConfiguration()).listFiles(path, recursive);
+} else {
+  return new ArrayList();
+}
+return getFiles(listStatus);
+  }
+
+  @Override
   public CarbonFile[] locationAwareListFiles() throws IOException {
 if (null != fileStatus && fileStatus.isDirectory()) {
   List listStatus = new ArrayList<>();
@@ -533,6 +545,9 @@ public abstract class AbstractDFSCarbonFile implements 
CarbonFile {
*/
   protected abstract CarbonFile[] getFiles(FileStatus[] listStatus);
 
+  protected abstract List 
getFiles(RemoteIterator listStatus)
+  throws IOException;
+
   @Override
   public String[] getLocations() throws IOException {
 BlockLocation[] blkLocations;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/13cdeb9f/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
index e0df6ae..61316f8 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
@@ -27,7 +27,9 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
 
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 
 
@@ -67,6 +69,16 @@ public class AlluxioCarbonFile extends AbstractDFSCarbonFile 
{
 return files;
   }
 
+  @Override
+  protected List getFiles(RemoteIterator 
listStatus)
+  throws IOException {
+List carbonFiles = new ArrayList<>();
+while (listStatus.hasNext()) {
+  Path filePath = listStatus.next().getPath();
+  carbonFiles.add(new AlluxioCarbonFile(filePath));
+}
+return carbonFiles;
+  }
 
   @Override
   public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/13cdeb9f/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java

Jenkins build became unstable: carbondata-master-spark-2.2 #280

2018-04-11 Thread Apache Jenkins Server

See

Jenkins build is unstable: carbondata-master-spark-2.2 » Apache CarbonData :: Spark Common Test #280

2018-04-11 Thread Apache Jenkins Server

See

carbondata git commit: [CARBONDATA-2329] Non Serializable extra info in session is overwritten from stale thread

2018-04-11 Thread manishgupta88

Repository: carbondata
Updated Branches:
  refs/heads/branch-1.3 4544765a4 -> 28f33746a


[CARBONDATA-2329] Non Serializable extra info in session is overwritten from 
stale thread

Problem:
1. Non Serializable extra info is copied from thread which causes stale data 
from old session when the thread is reused by spark.
2. CarboSessionInfo clone is not copying Non serializable info to new object 
which can damage session level values if local query thread updates values.

Solution: Remove logic to copy Non Serializable extra info and fix clone logic

This closes #2154


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/28f33746
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/28f33746
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/28f33746

Branch: refs/heads/branch-1.3
Commit: 28f33746a2feb8774cf40e8e392e9980c64d0292
Parents: 4544765
Author: KanakaKumar 
Authored: Tue Apr 10 19:42:30 2018 +0530
Committer: manishgupta88 
Committed: Wed Apr 11 14:33:44 2018 +0530

--
 .../org/apache/carbondata/core/util/CarbonSessionInfo.java | 6 ++
 .../spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala | 2 --
 .../src/main/scala/org/apache/spark/sql/CarbonSession.scala| 4 
 3 files changed, 2 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/28f33746/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java 
b/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java
index e397e55..3da44be 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java
@@ -74,11 +74,9 @@ public class CarbonSessionInfo implements Serializable, 
Cloneable {
 ex.printStackTrace();
   }
 }
-Map nonSerializableExtraInfo = 
getNonSerializableExtraInfo();
-for (Map.Entry entry : 
nonSerializableExtraInfo.entrySet()) {
-  nonSerializableExtraInfo.put(entry.getKey(), entry.getValue());
+for (Map.Entry entry : 
getNonSerializableExtraInfo().entrySet()) {
+  newObj.getNonSerializableExtraInfo().put(entry.getKey(), 
entry.getValue());
 }
-newObj.setNonSerializableExtraInfo(nonSerializableExtraInfo);
 return newObj;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/28f33746/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala 
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 4647bee..ed7e83b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -75,11 +75,9 @@ class CarbonEnv {
 // update carbon session parameters , preserve thread parameters
 val currentThreadSesssionInfo = 
ThreadLocalSessionInfo.getCarbonSessionInfo
 carbonSessionInfo = new CarbonSessionInfo()
-sessionParams = carbonSessionInfo.getSessionParams
 if (currentThreadSesssionInfo != null) {
   
carbonSessionInfo.setThreadParams(currentThreadSesssionInfo.getThreadParams)
 }
-carbonSessionInfo.setSessionParams(sessionParams)
 ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo)
 val config = new CarbonSQLConf(sparkSession)
 if 
(sparkSession.conf.getOption(CarbonCommonConstants.ENABLE_UNSAFE_SORT).isEmpty) 
{

http://git-wip-us.apache.org/repos/asf/carbondata/blob/28f33746/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala 
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index 0116d9e..e95b8db 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -246,11 +246,7 @@ object CarbonSession {
   // copy all the thread parameters to apply to session parameters
   currentThreadSessionInfo.getThreadParams.getAll.asScala
 .foreach(entry => 
carbonSessionInfo.getSessionParams.addProperty(entry._1, entry._2))
-  currentThreadSessionInfo.getNonSerializableExtraInfo.asScala
-.foreach(entry =>

Jenkins build is back to stable : carbondata-master-spark-2.1 #2238

2018-04-11 Thread Apache Jenkins Server

See

Jenkins build is back to stable : carbondata-master-spark-2.1 » Apache CarbonData :: Spark Common Test #2238

2018-04-11 Thread Apache Jenkins Server

See

carbondata git commit: [CARBONDATA-2321] Fix for selection of partion column after concurrent load fails randomly

2018-04-11 Thread manishgupta88

Repository: carbondata
Updated Branches:
  refs/heads/master 4cbd5cdf2 -> df8f06739


[CARBONDATA-2321] Fix for selection of partion column after concurrent load 
fails randomly

Fix selection of partition column after concurrent load fails randomly

This closes #2146


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/df8f0673
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/df8f0673
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/df8f0673

Branch: refs/heads/master
Commit: df8f06739d752d39863846eacb8947e667ce5a55
Parents: 4cbd5cd
Author: Jatin 
Authored: Sun Apr 8 15:35:19 2018 +0530
Committer: manishgupta88 
Committed: Wed Apr 11 14:17:59 2018 +0530

--
 .../core/writer/CarbonIndexFileMergeWriter.java   | 18 +-
 1 file changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/df8f0673/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
index bcecdef..ceeb431 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
@@ -80,8 +80,8 @@ public class CarbonIndexFileMergeWriter {
 }
 if (isCarbonIndexFilePresent(indexFiles) || indexFileNamesTobeAdded != 
null) {
   if (sfs == null) {
-return writeMergeIndexFileBasedOnSegmentFolder(
-indexFileNamesTobeAdded, readFileFooterFromCarbonDataFile, 
segmentPath, indexFiles);
+return writeMergeIndexFileBasedOnSegmentFolder(indexFileNamesTobeAdded,
+readFileFooterFromCarbonDataFile, segmentPath, indexFiles, 
segmentId);
   } else {
 return writeMergeIndexFileBasedOnSegmentFile(
 segmentId, indexFileNamesTobeAdded, sfs, indexFiles);
@@ -90,10 +90,9 @@ public class CarbonIndexFileMergeWriter {
 return null;
   }
 
-
   private String writeMergeIndexFileBasedOnSegmentFolder(List 
indexFileNamesTobeAdded,
-  boolean readFileFooterFromCarbonDataFile, String segmentPath, 
CarbonFile[] indexFiles)
-  throws IOException {
+  boolean readFileFooterFromCarbonDataFile, String segmentPath, 
CarbonFile[] indexFiles,
+  String segmentId) throws IOException {
 SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
 if (readFileFooterFromCarbonDataFile) {
   // this case will be used in case of upgrade where old store will not 
have the blocklet
@@ -104,7 +103,7 @@ public class CarbonIndexFileMergeWriter {
   fileStore.readAllIIndexOfSegment(segmentPath);
 }
 Map indexMap = fileStore.getCarbonIndexMap();
-writeMergeIndexFile(indexFileNamesTobeAdded, segmentPath, indexMap);
+writeMergeIndexFile(indexFileNamesTobeAdded, segmentPath, indexMap, 
segmentId);
 for (CarbonFile indexFile : indexFiles) {
   indexFile.delete();
 }
@@ -132,7 +131,7 @@ public class CarbonIndexFileMergeWriter {
 }
 for (Map.Entry> entry : 
indexLocationMap.entrySet()) {
   String mergeIndexFile =
-  writeMergeIndexFile(indexFileNamesTobeAdded, entry.getKey(), 
entry.getValue());
+  writeMergeIndexFile(indexFileNamesTobeAdded, entry.getKey(), 
entry.getValue(), segmentId);
   for (Map.Entry segentry : 
sfs.getLocationMap()
   .entrySet()) {
 String location = segentry.getKey();
@@ -163,7 +162,7 @@ public class CarbonIndexFileMergeWriter {
   }
 
   private String writeMergeIndexFile(List indexFileNamesTobeAdded, 
String segmentPath,
-  Map indexMap) throws IOException {
+  Map indexMap, String segment_id) throws IOException {
 MergedBlockIndexHeader indexHeader = new MergedBlockIndexHeader();
 MergedBlockIndex mergedBlockIndex = new MergedBlockIndex();
 List fileNames = new ArrayList<>(indexMap.size());
@@ -176,7 +175,8 @@ public class CarbonIndexFileMergeWriter {
   }
 }
 if (fileNames.size() > 0) {
-  String mergeIndexName = System.currentTimeMillis() + 
CarbonTablePath.MERGE_INDEX_FILE_EXT;
+  String mergeIndexName =
+  segment_id + '_' + System.currentTimeMillis() + 
CarbonTablePath.MERGE_INDEX_FILE_EXT;
   openThriftWriter(segmentPath + "/" + mergeIndexName);
   indexHeader.setFile_names(fileNames);
   mergedBlockIndex.setFileData(data);

carbondata git commit: [CARBONDATA-2321] Fix selection of partition column after concurrent load ails randomly

2018-04-11 Thread manishgupta88

Repository: carbondata
Updated Branches:
  refs/heads/branch-1.3 cc2a74e65 -> 4544765a4


[CARBONDATA-2321] Fix selection of partition column after concurrent load ails 
randomly

Fix selection of partition column after concurrent load fails randomly

This closes #2155


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4544765a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4544765a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4544765a

Branch: refs/heads/branch-1.3
Commit: 4544765a486b3fb1fbd4513992fbd79d3f9565ea
Parents: cc2a74e
Author: Jatin 
Authored: Wed Apr 11 12:04:31 2018 +0530
Committer: manishgupta88 
Committed: Wed Apr 11 14:13:54 2018 +0530

--
 .../core/writer/CarbonIndexFileMergeWriter.java   | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/4544765a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
index bc150e5..e618f7f 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
@@ -71,9 +71,9 @@ public class CarbonIndexFileMergeWriter {
 if (isCarbonIndexFilePresent(indexFiles) || indexFileNamesTobeAdded != 
null) {
   if (sfs == null) {
 return mergeNormalSegment(indexFileNamesTobeAdded, 
readFileFooterFromCarbonDataFile,
-segmentPath, indexFiles);
+segmentPath, indexFiles, segmentId);
   } else {
-return mergePartitionSegment(indexFileNamesTobeAdded, sfs, indexFiles);
+return mergePartitionSegment(indexFileNamesTobeAdded, sfs, indexFiles, 
segmentId);
   }
 }
 return null;
@@ -81,7 +81,8 @@ public class CarbonIndexFileMergeWriter {
 
 
   private SegmentIndexFIleMergeStatus mergeNormalSegment(List 
indexFileNamesTobeAdded,
-  boolean readFileFooterFromCarbonDataFile, String segmentPath, 
CarbonFile[] indexFiles)
+  boolean readFileFooterFromCarbonDataFile, String segmentPath, 
CarbonFile[] indexFiles,
+  String segmentId)
   throws IOException {
 SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
 if (readFileFooterFromCarbonDataFile) {
@@ -93,7 +94,7 @@ public class CarbonIndexFileMergeWriter {
   fileStore.readAllIIndexOfSegment(segmentPath);
 }
 Map indexMap = fileStore.getCarbonIndexMap();
-writeMergeIndexFile(indexFileNamesTobeAdded, segmentPath, indexMap);
+writeMergeIndexFile(indexFileNamesTobeAdded, segmentPath, indexMap, 
segmentId);
 for (CarbonFile indexFile : indexFiles) {
   indexFile.delete();
 }
@@ -101,7 +102,7 @@ public class CarbonIndexFileMergeWriter {
   }
 
   private SegmentIndexFIleMergeStatus mergePartitionSegment(List 
indexFileNamesTobeAdded,
-  SegmentFileStore sfs, CarbonFile[] indexFiles) throws IOException {
+  SegmentFileStore sfs, CarbonFile[] indexFiles, String segmentId) throws 
IOException {
 SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
 fileStore
 .readAllIIndexOfSegment(sfs.getSegmentFile(), sfs.getTablePath(), 
SegmentStatus.SUCCESS,
@@ -119,7 +120,7 @@ public class CarbonIndexFileMergeWriter {
 }
 for (Map.Entry> entry : 
indexLocationMap.entrySet()) {
   String mergeIndexFile =
-  writeMergeIndexFile(indexFileNamesTobeAdded, entry.getKey(), 
entry.getValue());
+  writeMergeIndexFile(indexFileNamesTobeAdded, entry.getKey(), 
entry.getValue(), segmentId);
   for (Map.Entry segentry : 
sfs.getLocationMap()
   .entrySet()) {
 String location = segentry.getKey();
@@ -141,7 +142,7 @@ public class CarbonIndexFileMergeWriter {
   }
 
   private String writeMergeIndexFile(List indexFileNamesTobeAdded, 
String segmentPath,
-  Map indexMap) throws IOException {
+  Map indexMap, String segmentId) throws IOException {
 MergedBlockIndexHeader indexHeader = new MergedBlockIndexHeader();
 MergedBlockIndex mergedBlockIndex = new MergedBlockIndex();
 List fileNames = new ArrayList<>(indexMap.size());
@@ -154,7 +155,8 @@ public class CarbonIndexFileMergeWriter {
   }
 }
 if (fileNames.size() > 0) {
-  String mergeIndexName = System.currentTimeMillis() + 
CarbonTablePath.MERGE_INDEX_FILE_EXT;
+

Jenkins build is back to stable : carbondata-master-spark-2.2 #278

2018-04-11 Thread Apache Jenkins Server

See

Jenkins build is back to stable : carbondata-master-spark-2.2 » Apache CarbonData :: Spark Common Test #278

2018-04-11 Thread Apache Jenkins Server

See

carbondata git commit: [CARBONDATA-2317] Concurrent datamap with same name and schema creation throws exception

2018-04-11 Thread manishgupta88

Repository: carbondata
Updated Branches:
  refs/heads/branch-1.3 3c48df396 -> cc2a74e65


[CARBONDATA-2317] Concurrent datamap with same name and schema creation throws 
exception

Concurrent datamap with same name and schema creation throws exception

This closes #2156


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/cc2a74e6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/cc2a74e6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/cc2a74e6

Branch: refs/heads/branch-1.3
Commit: cc2a74e65001ee73da4fad47852984d4f1c42adf
Parents: 3c48df3
Author: rahulforallp 
Authored: Fri Apr 6 15:17:54 2018 +0530
Committer: manishgupta88 
Committed: Wed Apr 11 13:55:41 2018 +0530

--
 .../preaggregate/TestPreAggCreateCommand.scala  | 43 
 .../table/CarbonCreateTableCommand.scala| 27 ++--
 2 files changed, 58 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/cc2a74e6/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index 5b7c310..5813211 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -17,7 +17,14 @@
 
 package org.apache.carbondata.integration.spark.testsuite.preaggregate
 
+import java.util
+import java.util.concurrent.{Callable, ExecutorService, Executors, TimeUnit}
+
 import scala.collection.JavaConverters._
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.{Await, Future}
+import scala.concurrent.duration.Duration
+import scala.util.{Failure, Success}
 
 import org.apache.spark.sql.{AnalysisException, 
CarbonDatasourceHadoopRelation, Row}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
@@ -415,6 +422,42 @@ class TestPreAggCreateCommand extends QueryTest with 
BeforeAndAfterAll {
 sql("use default")
   }
 
+  test("test creation of multiple preaggregate of same name concurrently ") {
+sql("DROP TABLE IF EXISTS tbl_concurr")
+sql(
+  "create table if not exists  tbl_concurr(imei string,age int,mac string 
,prodate timestamp," +
+  "update timestamp,gamepoint double,contrid double) stored by 
'carbondata' ")
+
+var executorService: ExecutorService = Executors.newCachedThreadPool()
+val tasks = new util.ArrayList[Callable[String]]()
+var i = 0
+val count = 5
+while (i < count) {
+  tasks
+.add(new QueryTask(
+  s"""create datamap agg_concu1 on table tbl_concurr using
+ |'preaggregate' as select prodate, mac from tbl_concurr group by 
prodate,mac"""
+.stripMargin))
+  i = i + 1
+}
+executorService.invokeAll(tasks)
+
+checkExistence(sql("show tables"), true, "agg_concu1", "tbl_concurr")
+executorService.shutdown()
+  }
+
+  class QueryTask(query: String) extends Callable[String] {
+override def call(): String = {
+  var result = "SUCCESS"
+  try {
+sql(query).collect()
+  } catch {
+case exception: Exception => LOGGER.error(exception.getMessage)
+  }
+  result
+}
+  }
+
   def getCarbontable(plan: LogicalPlan) : CarbonTable ={
 var carbonTable : CarbonTable = null
 plan.transform {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/cc2a74e6/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
index 22dab27..e05cda6 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
@@ -119,19 +119,22 @@ case class CarbonCreateTableCommand(
   // isVisible property is added to hive table properties to 
differentiate between