[30/54] [abbrv] carbondata git commit: [CARBONDATA-2172][Lucene] Add text_columns property for Lucene DataMap

[ravipesala]

[CARBONDATA-2172][Lucene] Add text_columns property for Lucene DataMap

Add text_columns property for Lucene DataMap

This closes #2019


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d23f7fad
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d23f7fad
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d23f7fad

Branch: refs/heads/master
Commit: d23f7fad1f7db029d1dd0cc8e3db7a5b79463179
Parents: f9291cd
Author: QiangCai 
Authored: Thu Mar 1 15:40:01 2018 +0800
Committer: ravipesala 
Committed: Thu Mar 8 22:21:11 2018 +0530

--
 .../core/datamap/DataMapStoreManager.java   | 16 ++--
 .../core/datamap/dev/DataMapFactory.java|  4 +-
 .../blockletindex/BlockletDataMapFactory.java   |  2 +-
 .../ThriftWrapperSchemaConverterImpl.java   |  2 +-
 .../schema/datamap/DataMapProvider.java |  4 +
 .../schema/table/AggregationDataMapSchema.java  |  2 +-
 .../core/metadata/schema/table/CarbonTable.java | 16 +++-
 .../metadata/schema/table/DataMapSchema.java| 44 --
 .../core/metadata/schema/table/TableInfo.java   |  2 +-
 .../core/metadata/schema/table/TableSchema.java | 16 ++--
 datamap/lucene/pom.xml  |  5 ++
 .../lucene/LuceneDataMapFactoryBase.java| 89 ++--
 .../lucene/LuceneFineGrainDataMapSuite.scala| 58 -
 .../preaggregate/TestPreAggCreateCommand.scala  | 75 -
 .../preaggregate/TestPreAggregateLoad.scala |  7 +-
 .../timeseries/TestTimeseriesDataLoad.scala |  6 +-
 .../testsuite/datamap/CGDataMapTestCase.scala   |  4 +-
 .../testsuite/datamap/DataMapWriterSuite.scala  |  1 +
 .../testsuite/datamap/FGDataMapTestCase.scala   |  4 +-
 .../iud/InsertOverwriteConcurrentTest.scala |  0
 .../TestInsertAndOtherCommandConcurrent.scala   | 38 -
 .../carbondata/spark/load/ValidateUtil.scala|  0
 .../carbondata/spark/util/DataLoadingUtil.scala |  0
 .../carbondata/datamap/DataMapManager.java  |  4 +-
 .../datamap/IndexDataMapProvider.java   |  6 +-
 .../datamap/PreAggregateDataMapProvider.java|  2 +-
 .../datamap/TimeseriesDataMapProvider.java  |  4 +-
 .../datamap/CarbonCreateDataMapCommand.scala| 21 +++--
 .../datamap/CarbonDataMapShowCommand.scala  |  2 +-
 .../preaaggregate/PreAggregateListeners.scala   |  9 +-
 .../command/timeseries/TimeSeriesUtil.scala | 16 ++--
 .../loading/model/CarbonLoadModelBuilder.java   | 16 ++--
 .../processing/loading/model/LoadOption.java| 23 -
 33 files changed, 311 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index ab339e8..a8d467f 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -33,7 +33,6 @@ import 
org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
 import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
 import 
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
 import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
@@ -89,8 +88,7 @@ public final class DataMapStoreManager {
 List dataMaps = new ArrayList<>();
 if (dataMapSchemaList != null) {
   for (DataMapSchema dataMapSchema : dataMapSchemaList) {
-if (!dataMapSchema.getClassName().equalsIgnoreCase(
-DataMapProvider.PREAGGREGATE.toString())) {
+if (dataMapSchema.isIndexDataMap()) {
   dataMaps.add(getDataMap(carbonTable.getAbsoluteTableIdentifier(), 
dataMapSchema));
 }
   }
@@ -144,26 +142,28 @@ public final class DataMapStoreManager {
* Return a new datamap instance and registered in the store manager.
* The datamap is created using datamap name, datamap factory class and 
table identifier.
*/
-  private TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier 
identifier,
+  // TODO: make it private
+  public TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier 
identifier,
   DataMapSchema dataMapSchema) throws MalformedDataMapCommandException, 

carbondata git commit: [CARBONDATA-2172][Lucene] Add text_columns property for Lucene DataMap

[jackylk]

Repository: carbondata
Updated Branches:
  refs/heads/datamap 69eb26e68 -> 9390abf6a


[CARBONDATA-2172][Lucene] Add text_columns property for Lucene DataMap

Add text_columns property for Lucene DataMap

This closes #2019


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9390abf6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9390abf6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9390abf6

Branch: refs/heads/datamap
Commit: 9390abf6a0a79261b9de981502df9017aab68f96
Parents: 69eb26e
Author: QiangCai 
Authored: Thu Mar 1 15:40:01 2018 +0800
Committer: Jacky Li 
Committed: Mon Mar 5 10:52:51 2018 +0800

--
 .../core/datamap/DataMapStoreManager.java   |  3 +-
 .../core/datamap/dev/DataMapFactory.java|  4 +-
 datamap/lucene/pom.xml  |  5 ++
 .../lucene/LuceneDataMapFactoryBase.java| 89 ++--
 .../lucene/LuceneFineGrainDataMapSuite.scala| 58 -
 5 files changed, 126 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/9390abf6/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 0223ae2..e7c72e8 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -163,7 +163,8 @@ public final class DataMapStoreManager {
   }
 
   public TableDataMap registerDataMap(AbsoluteTableIdentifier identifier,
-  DataMapSchema dataMapSchema,  DataMapFactory dataMapFactory) throws 
IOException {
+  DataMapSchema dataMapSchema,  DataMapFactory dataMapFactory)
+  throws IOException, MalformedDataMapCommandException {
 String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
 // Just update the segmentRefreshMap with the table if not added.
 getTableSegmentRefresher(identifier);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9390abf6/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java 
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index 7bf04c9..ef9bb66 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.datamap.dev;
 import java.io.IOException;
 import java.util.List;
 
+import 
org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
 import org.apache.carbondata.core.datamap.DataMapDistributable;
 import org.apache.carbondata.core.datamap.DataMapLevel;
 import org.apache.carbondata.core.datamap.DataMapMeta;
@@ -34,7 +35,8 @@ public interface DataMapFactory {
   /**
* Initialization of Datamap factory with the identifier and datamap name
*/
-  void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) 
throws IOException;
+  void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema)
+  throws IOException, MalformedDataMapCommandException;
 
   /**
* Return a new write for this datamap

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9390abf6/datamap/lucene/pom.xml
--
diff --git a/datamap/lucene/pom.xml b/datamap/lucene/pom.xml
index ee504c6..4019065 100644
--- a/datamap/lucene/pom.xml
+++ b/datamap/lucene/pom.xml
@@ -26,6 +26,11 @@
   ${project.version}
 
 
+  org.apache.commons
+  commons-lang3
+  3.3.2
+
+
   org.apache.lucene
   lucene-core
   ${lucene.version}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9390abf6/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
--
diff --git 
a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
 
b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
index 5eb7054..3a1adab 100644
--- 
a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
+++