date:20180507

carbondata git commit: [CARBONDATA-2424] Added documentation for properties of Pre-aggregate tables

2018-05-07 Thread kunalkapoor

Repository: carbondata
Updated Branches:
  refs/heads/master b1c85fa55 -> 1128d89ea


[CARBONDATA-2424] Added documentation for properties of Pre-aggregate tables

'path' and 'partitioning' under DMPROPERTIES have been updated in the document

This closes #2258


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1128d89e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1128d89e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1128d89e

Branch: refs/heads/master
Commit: 1128d89eae927657f82842f1d8896f705befca76
Parents: b1c85fa
Author: praveenmeenakshi56 
Authored: Wed May 2 13:47:25 2018 +0530
Committer: kunal642 
Committed: Mon May 7 18:04:20 2018 +0530

--
 docs/datamap/preaggregate-datamap-guide.md | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/1128d89e/docs/datamap/preaggregate-datamap-guide.md
--
diff --git a/docs/datamap/preaggregate-datamap-guide.md 
b/docs/datamap/preaggregate-datamap-guide.md
index 84f06c4..ce7cbcc 100644
--- a/docs/datamap/preaggregate-datamap-guide.md
+++ b/docs/datamap/preaggregate-datamap-guide.md
@@ -105,7 +105,10 @@ DataMap can be created using following DDL
   ```
 The string followed by USING is called DataMap Provider, in this version 
CarbonData supports two 
 kinds of DataMap: 
-1. preaggregate, for pre-aggregate table. No DMPROPERTY is required for this 
DataMap
+1. preaggregate, for pre-aggregate table. Pre-Aggregate table supports two 
values for DMPROPERTIES.
+   a. 'path' is used to specify the store location of the 
datamap.('path'='/location/').
+   b. 'partitioning' when set to false enables user to disable partitioning of 
the datamap.
+   Default value is true for this property.
 2. timeseries, for timeseries roll-up table. Please refer to [Timeseries 
DataMap](https://github.com/apache/carbondata/blob/master/docs/datamap/timeseries-datamap-guide.md)
 
 DataMap can be dropped using following DDL

carbondata git commit: [CARBONDATA-2430][SDK] Reshuffling of Columns given by user in SDK.

2018-05-07 Thread ravipesala

Repository: carbondata
Updated Branches:
  refs/heads/master 531ecdf3f -> b1c85fa55


[CARBONDATA-2430][SDK] Reshuffling of Columns given by user in SDK.

Reshuffling of Columns given by the user in SDK. Order should be Sort COlumns 
-> Dimension -> Complex --> Measure

This closes #2261


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b1c85fa5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b1c85fa5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b1c85fa5

Branch: refs/heads/master
Commit: b1c85fa55eeca1a98b61117c6b46df9a28d60bca
Parents: 531ecdf
Author: sounakr 
Authored: Wed May 2 21:29:57 2018 +0530
Committer: ravipesala 
Committed: Mon May 7 16:36:46 2018 +0530

--
 .../core/metadata/datatype/ArrayType.java   |   4 +
 .../core/metadata/datatype/StructField.java |  15 +
 .../core/metadata/schema/table/CarbonTable.java |  26 --
 .../schema/table/TableSchemaBuilder.java|  72 ++-
 .../examples/DataFrameComplexTypeExample.scala  | 147 +-
 .../TestNonTransactionalCarbonTable.scala   | 459 +--
 .../apache/spark/util/SparkTypeConverter.scala  |  58 +--
 .../carbondata/sdk/file/AvroCarbonWriter.java   |  48 +-
 .../sdk/file/CarbonWriterBuilder.java   | 104 +++--
 .../org/apache/carbondata/sdk/file/Field.java   |   4 +-
 .../sdk/file/AvroCarbonWriterTest.java  | 214 +
 11 files changed, 963 insertions(+), 188 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b1c85fa5/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
 
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
index c30e21c..c327d7f 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
@@ -30,4 +30,8 @@ public class ArrayType extends DataType {
   public boolean isComplexType() {
 return true;
   }
+
+  public DataType getElementType() {
+return elementType;
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b1c85fa5/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
 
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
index efdc8e2..bfca057 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
@@ -18,6 +18,7 @@
 package org.apache.carbondata.core.metadata.datatype;
 
 import java.io.Serializable;
+import java.util.List;
 
 public class StructField implements Serializable {
 
@@ -27,9 +28,19 @@ public class StructField implements Serializable {
 
   private DataType dataType;
 
+  private List children;
+
   public StructField(String fieldName, DataType dataType) {
 this.fieldName = fieldName;
 this.dataType = dataType;
+this.children = null;
+  }
+
+
+  public StructField(String fieldName, DataType dataType, List 
children) {
+this.fieldName = fieldName;
+this.dataType = dataType;
+this.children = children;
   }
 
   public DataType getDataType() {
@@ -39,4 +50,8 @@ public class StructField implements Serializable {
   public String getFieldName() {
 return fieldName;
   }
+
+  public List getChildren() {
+return children;
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b1c85fa5/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 4178d8a..cf5660f 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -37,8 +37,6 @@ import org.apache.carbondata.core.datamap.dev.DataMapFactory;
 import org.apache.carbondata.core.features.TableOperation;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import 
org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
-import

[2/2] carbondata git commit: [CARBONDATA-2310] Refactored code to improve Distributable interface & [CARBONDATA-2362] Changing the Cacheable object from DataMap to Wrapper

2018-05-07 Thread ravipesala

[CARBONDATA-2310] Refactored code to improve Distributable interface & 
[CARBONDATA-2362] Changing the Cacheable object from DataMap to Wrapper

This PR has two JIRA fixes
[CARBONDATA-2310] Refactored code to improve Distributable interface
[CARBONDATA-2362] Changing the Cacheable object from DataMap to Wrapper

This closes #2244


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/531ecdf3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/531ecdf3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/531ecdf3

Branch: refs/heads/master
Commit: 531ecdf3f40c064d4ff6ad36c43fa90a2d423588
Parents: a7926ea
Author: dhatchayani 
Authored: Fri Apr 27 23:03:52 2018 +0530
Committer: ravipesala 
Committed: Mon May 7 13:11:29 2018 +0530

--
 .../org/apache/carbondata/core/cache/Cache.java |  10 +
 .../dictionary/AbstractDictionaryCache.java |   6 +
 .../core/constants/CarbonCommonConstants.java   |   3 +
 .../core/datamap/dev/CacheableDataMap.java  |  51 
 .../core/datastore/SegmentTaskIndexStore.java   |   7 +
 .../filesystem/AbstractDFSCarbonFile.java   |   5 +-
 .../core/datastore/filesystem/CarbonFile.java   |   3 +-
 .../datastore/filesystem/LocalCarbonFile.java   |   3 +-
 .../core/indexstore/AbstractMemoryDMStore.java  |  63 +
 .../indexstore/BlockletDataMapIndexStore.java   | 187 ---
 .../indexstore/BlockletDataMapIndexWrapper.java |  52 +
 .../core/indexstore/BlockletDetailInfo.java |  66 --
 .../core/indexstore/SafeMemoryDMStore.java  | 105 +
 .../TableBlockIndexUniqueIdentifier.java|   5 +-
 .../core/indexstore/UnsafeMemoryDMStore.java|  25 +-
 .../blockletindex/BlockletDataMap.java  | 232 ---
 .../BlockletDataMapDistributable.java   |  12 +
 .../blockletindex/BlockletDataMapFactory.java   | 127 ++
 .../blockletindex/BlockletDataMapModel.java |  12 +
 .../blockletindex/SegmentIndexFileStore.java|  39 +++-
 .../core/indexstore/row/DataMapRow.java |  13 +-
 .../core/indexstore/row/UnsafeDataMapRow.java   |   7 +-
 .../core/indexstore/schema/CarbonRowSchema.java |  10 +-
 .../core/metadata/SegmentFileStore.java |  29 +++
 .../core/metadata/schema/table/TableInfo.java   |  24 ++
 .../TableStatusReadCommittedScope.java  |   4 +-
 .../core/util/BlockletDataMapUtil.java  | 180 ++
 .../carbondata/core/util/SessionParams.java |   5 +
 .../core/util/path/CarbonTablePath.java |   2 +-
 .../TestBlockletDataMapFactory.java | 126 ++
 .../apache/carbondata/hadoop/CacheClient.java   |  49 
 .../hadoop/api/AbstractDataMapJob.java  |  42 
 .../hadoop/api/CarbonFileInputFormat.java   |   2 +-
 .../hadoop/api/CarbonInputFormat.java   |  27 ++-
 .../hadoop/api/CarbonTableInputFormat.java  |   2 +-
 .../carbondata/hadoop/api/DataMapJob.java   |   6 +
 .../hadoop/util/CarbonInputFormatUtil.java  |  43 +++-
 .../lucene/LuceneFineGrainDataMapSuite.scala|   1 +
 .../carbondata/spark/rdd/CarbonScanRDD.scala|  15 +-
 .../carbondata/spark/rdd/SparkDataMapJob.scala  |   4 +-
 .../org/apache/spark/sql/CarbonCountStar.scala  |   3 +
 .../execution/command/CarbonHiveCommands.scala  |   9 +
 42 files changed, 1335 insertions(+), 281 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/531ecdf3/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
--
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/Cache.java 
b/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
index 04fa18a..6df36fc 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
@@ -20,6 +20,8 @@ package org.apache.carbondata.core.cache;
 import java.io.IOException;
 import java.util.List;
 
+import org.apache.carbondata.core.memory.MemoryException;
+
 /**
  * A semi-persistent mapping from keys to values. Cache entries are manually 
added using
  * #get(Key), #getAll(List) , and are stored in the cache until
@@ -69,6 +71,14 @@ public interface Cache {
   void invalidate(K key);
 
   /**
+   * This method will add the value to the cache for the given key
+   *
+   * @param key
+   * @param value
+   */
+  void put(K key, V value) throws IOException, MemoryException;
+
+  /**
* Access count of Cacheable entry will be decremented
*
* @param keys

http://git-wip-us.apache.org/repos/asf/carbondata/blob/531ecdf3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java

[1/2] carbondata git commit: [CARBONDATA-2310] Refactored code to improve Distributable interface & [CARBONDATA-2362] Changing the Cacheable object from DataMap to Wrapper

2018-05-07 Thread ravipesala

Repository: carbondata
Updated Branches:
  refs/heads/master a7926ea13 -> 531ecdf3f


http://git-wip-us.apache.org/repos/asf/carbondata/blob/531ecdf3/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
 
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
index 6803fc8..c6efd77 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
@@ -70,9 +70,15 @@ public class SegmentIndexFileStore {
*/
   private Map carbonIndexMapWithFullPath;
 
+  /**
+   * Stores the list of index files in a merge file
+   */
+  private Map carbonMergeFileToIndexFilesMap;
+
   public SegmentIndexFileStore() {
 carbonIndexMap = new HashMap<>();
 carbonIndexMapWithFullPath = new HashMap<>();
+carbonMergeFileToIndexFilesMap = new HashMap<>();
   }
 
   /**
@@ -201,6 +207,28 @@ public class SegmentIndexFileStore {
   }
 
   /**
+   * Read all index file names of the segment
+   *
+   * @param segmentPath
+   * @return
+   * @throws IOException
+   */
+  public Map getMergeOrIndexFilesFromSegment(String 
segmentPath)
+  throws IOException {
+CarbonFile[] carbonIndexFiles = getCarbonIndexFiles(segmentPath);
+Map indexFiles = new HashMap<>();
+for (int i = 0; i < carbonIndexFiles.length; i++) {
+  if 
(carbonIndexFiles[i].getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
+indexFiles
+.put(carbonIndexFiles[i].getAbsolutePath(), 
carbonIndexFiles[i].getAbsolutePath());
+  } else if 
(carbonIndexFiles[i].getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
+indexFiles.put(carbonIndexFiles[i].getAbsolutePath(), null);
+  }
+}
+return indexFiles;
+  }
+
+  /**
* List all the index files inside merge file.
* @param mergeFile
* @return
@@ -221,13 +249,14 @@ public class SegmentIndexFileStore {
* @param mergeFilePath
* @throws IOException
*/
-  private void readMergeFile(String mergeFilePath) throws IOException {
+  public void readMergeFile(String mergeFilePath) throws IOException {
 ThriftReader thriftReader = new ThriftReader(mergeFilePath);
 try {
   thriftReader.open();
   MergedBlockIndexHeader indexHeader = 
readMergeBlockIndexHeader(thriftReader);
   MergedBlockIndex mergedBlockIndex = readMergeBlockIndex(thriftReader);
   List file_names = indexHeader.getFile_names();
+  carbonMergeFileToIndexFilesMap.put(mergeFilePath, file_names);
   List fileData = mergedBlockIndex.getFileData();
   CarbonFile mergeFile = FileFactory.getCarbonFile(mergeFilePath);
   assert (file_names.size() == fileData.size());
@@ -298,8 +327,8 @@ public class SegmentIndexFileStore {
 CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
 return carbonFile.listFiles(new CarbonFileFilter() {
   @Override public boolean accept(CarbonFile file) {
-return file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT) || 
file.getName()
-.endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT);
+return ((file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT) || 
file.getName()
+.endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) && file.getSize() 
> 0);
   }
 });
   }
@@ -428,4 +457,8 @@ public class SegmentIndexFileStore {
 + " is " + (System.currentTimeMillis() - startTime));
 return carbondataFileFooter.getBlockletList();
   }
+
+  public Map getCarbonMergeFileToIndexFilesMap() {
+return carbonMergeFileToIndexFilesMap;
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/531ecdf3/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java 
b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
index b764bdf..496a1d0 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
@@ -16,13 +16,15 @@
  */
 package org.apache.carbondata.core.indexstore.row;
 
+import java.io.Serializable;
+
 import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema;
 
 /**
  * It is just a normal row to store data. Implementation classes could be safe 
and unsafe.
  * TODO move this class a global row and use across loading after DataType is 
changed class
  */
-public abstract class DataMapRow {
+public abstract class DataMapRow

carbondata git commit: [CARBONDATA-2424] Added documentation for properties of Pre-aggregate tables

carbondata git commit: [CARBONDATA-2430][SDK] Reshuffling of Columns given by user in SDK.

[2/2] carbondata git commit: [CARBONDATA-2310] Refactored code to improve Distributable interface & [CARBONDATA-2362] Changing the Cacheable object from DataMap to Wrapper

[1/2] carbondata git commit: [CARBONDATA-2310] Refactored code to improve Distributable interface & [CARBONDATA-2362] Changing the Cacheable object from DataMap to Wrapper

4 matches

Site Navigation

Mail list logo

Footer information