carbondata git commit: [CARBONDATA-2482] Pass uuid while writing segment file if possible

[ravipesala]

Repository: carbondata
Updated Branches:
  refs/heads/master eb604fdb7 -> 6c5abddfb


[CARBONDATA-2482] Pass uuid while writing segment file if possible

Pass the uuid (segmentFileName) to the writeSegmentFile method file if possible.

Problem:
When the supporting tables depends on the segmentFileName of the main table, 
query is failing, as it is expected to be the same name as that of the main 
table. Mostly in case of merge index, segmentFile will be rewritten for that 
segment.

Solution:
Whenever the supporting table segmentFileName should be same as that main 
table, in that case we should pass the name as the UUID in the merge index flow 
instead of taking a new timestamp.

This closes #2307


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6c5abddf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6c5abddf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6c5abddf

Branch: refs/heads/master
Commit: 6c5abddfb125b2dd51c9a989f003fe2bdc066d6d
Parents: eb604fd
Author: dhatchayani 
Authored: Tue May 15 12:24:01 2018 +0530
Committer: ravipesala 
Committed: Mon Jul 16 10:34:31 2018 +0530

--
 .../core/datamap/DataMapStoreManager.java   |  3 +-
 .../core/writer/CarbonIndexFileMergeWriter.java | 40 ++--
 .../sdv/generated/MergeIndexTestCase.scala  |  8 ++--
 .../CarbonIndexFileMergeTestCase.scala  | 18 -
 .../TestStreamingTableWithRowParser.scala   |  2 +-
 .../store/CarbonFactDataHandlerModel.java   |  3 +-
 .../processing/util/CarbonLoaderUtil.java   | 10 +++--
 7 files changed, 35 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c5abddf/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 8ce302b..475ec01 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -584,7 +584,8 @@ public final class DataMapStoreManager {
   SegmentRefreshInfo segmentRefreshInfo =
   seg.getSegmentRefreshInfo(updateVo);
   String segmentId = seg.getSegmentNo();
-  if (segmentRefreshTime.get(segmentId) == null) {
+  if (segmentRefreshTime.get(segmentId) == null
+  && segmentRefreshInfo.getSegmentUpdatedTimestamp() != null) {
 segmentRefreshTime.put(segmentId, segmentRefreshInfo);
 return true;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c5abddf/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
index 80a46cb..b080f52 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
@@ -67,7 +67,7 @@ public class CarbonIndexFileMergeWriter {
*/
   private String mergeCarbonIndexFilesOfSegment(String segmentId,
   String tablePath, List indexFileNamesTobeAdded,
-  boolean readFileFooterFromCarbonDataFile) throws IOException {
+  boolean readFileFooterFromCarbonDataFile, String uuid) throws 
IOException {
 Segment segment = Segment.getSegment(segmentId, tablePath);
 String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
 CarbonFile[] indexFiles;
@@ -85,7 +85,7 @@ public class CarbonIndexFileMergeWriter {
 readFileFooterFromCarbonDataFile, segmentPath, indexFiles, 
segmentId);
   } else {
 return writeMergeIndexFileBasedOnSegmentFile(
-segmentId, indexFileNamesTobeAdded, sfs, indexFiles);
+segmentId, indexFileNamesTobeAdded, sfs, indexFiles, uuid);
   }
 }
 return null;
@@ -111,10 +111,9 @@ public class CarbonIndexFileMergeWriter {
 return null;
   }
 
-  private String writeMergeIndexFileBasedOnSegmentFile(
-  String segmentId,
-  List indexFileNamesTobeAdded,
-  SegmentFileStore segmentFileStore, CarbonFile[] indexFiles) throws 
IOException {
+  private String writeMergeIndexFileBasedOnSegmentFile(String segmentId,
+  List indexFileNamesTobeAdded, SegmentFileStore segmentFileStore,
+  CarbonFile[] indexFiles, String uuid) throws IOException {
 SegmentIndexFileStore fileStore = new 

Jenkins build became unstable: carbondata-master-spark-2.1 #2625

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Spark Common Test #2625

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.1 #2624

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.1 » Apache CarbonData :: Spark2 #2624

[Apache Jenkins Server]

See 

carbondata git commit: [CARBONDATA-2704] Index file size in describe formatted command is not updated correctly with the segment file

[manishgupta88]

Repository: carbondata
Updated Branches:
  refs/heads/master cdee81d4d -> eb604fdb7


[CARBONDATA-2704] Index file size in describe formatted command is not updated 
correctly with the segment file

Problem:
Describe formatted command is not showing correct index files size after index 
files merge.
Solution:
Segment file should be updated with the actual index files size of that segment 
after index files merge.

This closes #2462


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/eb604fdb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/eb604fdb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/eb604fdb

Branch: refs/heads/master
Commit: eb604fdb73983dfe9396d488a51907d90ed51d3e
Parents: cdee81d
Author: dhatchayani 
Authored: Mon Jul 9 11:19:51 2018 +0530
Committer: manishgupta88 
Committed: Sun Jul 15 20:34:32 2018 +0530

--
 .../core/metadata/SegmentFileStore.java |  4 +-
 .../apache/carbondata/core/util/CarbonUtil.java | 48 ---
 .../core/writer/CarbonIndexFileMergeWriter.java | 17 +++
 .../CarbonIndexFileMergeTestCase.scala  | 50 
 .../spark/rdd/CarbonDataRDDFactory.scala|  4 +-
 5 files changed, 96 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/eb604fdb/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java 
b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
index 3d3b245..ce79e65 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -281,7 +281,7 @@ public class SegmentFileStore {
* @throws IOException
*/
   public static boolean updateSegmentFile(String tablePath, String segmentId, 
String segmentFile,
-  String tableId) throws IOException {
+  String tableId, SegmentFileStore segmentFileStore) throws IOException {
 boolean status = false;
 String tableStatusPath = CarbonTablePath.getTableStatusFilePath(tablePath);
 if (!FileFactory.isFileExist(tableStatusPath)) {
@@ -308,6 +308,8 @@ public class SegmentFileStore {
   // if the segments is in the list of marked for delete then update 
the status.
   if (segmentId.equals(detail.getLoadName())) {
 detail.setSegmentFile(segmentFile);
+detail.setIndexSize(String.valueOf(CarbonUtil
+.getCarbonIndexSize(segmentFileStore, 
segmentFileStore.getLocationMap(;
 break;
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/eb604fdb/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
--
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java 
b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index e87e52c..9796696 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2647,23 +2647,7 @@ public final class CarbonUtil {
   fileStore.readIndexFiles();
   Map> indexFilesMap = fileStore.getIndexFilesMap();
   // get the size of carbonindex file
-  for (Map.Entry entry : 
locationMap.entrySet()) {
-SegmentFileStore.FolderDetails folderDetails = entry.getValue();
-Set carbonindexFiles = folderDetails.getFiles();
-String mergeFileName = folderDetails.getMergeFileName();
-if (null != mergeFileName) {
-  String mergeIndexPath =
-  fileStore.getTablePath() + entry.getKey() + 
CarbonCommonConstants.FILE_SEPARATOR
-  + mergeFileName;
-  carbonIndexSize += 
FileFactory.getCarbonFile(mergeIndexPath).getSize();
-}
-for (String indexFile : carbonindexFiles) {
-  String indexPath =
-  fileStore.getTablePath() + entry.getKey() + 
CarbonCommonConstants.FILE_SEPARATOR
-  + indexFile;
-  carbonIndexSize += FileFactory.getCarbonFile(indexPath).getSize();
-}
-  }
+  carbonIndexSize = getCarbonIndexSize(fileStore, locationMap);
   for (Map.Entry> entry : indexFilesMap.entrySet()) {
 // get the size of carbondata files
 for (String blockFile : entry.getValue()) {
@@ -2676,6 +2660,36 @@ public final class CarbonUtil {
 return dataAndIndexSize;
   }
 
+  /**
+   * Calcuate the index files size of the segment
+   *
+   * @param fileStore
+   * @param locationMap
+   * @return
+   */
+  

carbondata git commit: [CARBONDATA-2693][BloomDataMap]Fix bug for alter rename is renaming the existing table on which bloomfilter datamp exists

[jackylk]

Repository: carbondata
Updated Branches:
  refs/heads/master 75a602d01 -> cdee81d4d


[CARBONDATA-2693][BloomDataMap]Fix bug for alter rename is renaming the 
existing table on which bloomfilter datamp exists

Fix bug for alter rename is renaming the existing table on which bloom filter 
datamap exists

This closes #2452


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/cdee81d4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/cdee81d4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/cdee81d4

Branch: refs/heads/master
Commit: cdee81d4dbdd22ed728f2bd898ed888ed25a8774
Parents: 75a602d
Author: ndwangsen 
Authored: Thu Jul 5 16:51:39 2018 +0800
Committer: Jacky Li 
Committed: Sun Jul 15 21:44:53 2018 +0800

--
 .../core/datamap/DataMapStoreManager.java   |  51 -
 .../core/metadata/schema/table/CarbonTable.java |  24 +++
 .../table/DiskBasedDMSchemaStorageProvider.java |   3 +-
 .../bloom/BloomCoarseGrainDataMapFactory.java   |  21 +-
 .../TestRenameTableWithDataMap.scala| 192 +++
 .../preaaggregate/PreAggregateListeners.scala   |   5 -
 .../schema/CarbonAlterTableRenameCommand.scala  |  22 ++-
 7 files changed, 306 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/cdee81d4/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 9a7d1c1..8ce302b 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -50,6 +50,9 @@ import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.CarbonSessionInfo;
 import org.apache.carbondata.core.util.ThreadLocalSessionInfo;
 
+import static 
org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.MV;
+import static 
org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.PREAGGREGATE;
+
 /**
  * It maintains all the DataMaps in it.
  */
@@ -125,9 +128,8 @@ public final class DataMapStoreManager {
 if (dataMapSchemas != null) {
   for (DataMapSchema dataMapSchema : dataMapSchemas) {
 RelationIdentifier identifier = dataMapSchema.getParentTables().get(0);
-if (dataMapSchema.isIndexDataMap() && identifier.getTableName()
-.equals(carbonTable.getTableName()) && identifier.getDatabaseName()
-.equals(carbonTable.getDatabaseName())) {
+if (dataMapSchema.isIndexDataMap() && identifier.getTableId()
+.equals(carbonTable.getTableId())) {
   dataMaps.add(getDataMap(carbonTable, dataMapSchema));
 }
   }
@@ -173,6 +175,49 @@ public final class DataMapStoreManager {
   }
 
   /**
+   * Update the datamap schema after table rename
+   * This should be invoked after changing table name
+   * @param dataMapSchemaList
+   * @param newTableName
+   */
+  public void updateDataMapSchema(List dataMapSchemaList,
+  String newTableName) throws IOException {
+List newDataMapSchemas = new ArrayList<>();
+for (DataMapSchema dataMapSchema : dataMapSchemaList) {
+  RelationIdentifier relationIdentifier = 
dataMapSchema.getRelationIdentifier();
+  String dataBaseName =  relationIdentifier.getDatabaseName();
+  String tableId = relationIdentifier.getTableId();
+  String providerName = dataMapSchema.getProviderName();
+  // if the preaggregate datamap,not be modified the schema
+  if (providerName.equalsIgnoreCase(PREAGGREGATE.toString())) {
+continue;
+  }
+  // if the mv datamap,not be modified the relationIdentifier
+  if (!providerName.equalsIgnoreCase(MV.toString())) {
+RelationIdentifier newRelationIdentifier = new 
RelationIdentifier(dataBaseName,
+newTableName, tableId);
+dataMapSchema.setRelationIdentifier(newRelationIdentifier);
+  }
+  List newParentTables = new ArrayList<>();
+  List parentTables = dataMapSchema.getParentTables();
+  for (RelationIdentifier identifier : parentTables) {
+RelationIdentifier newParentTableIdentifier = new RelationIdentifier(
+identifier.getDatabaseName(), newTableName, 
identifier.getTableId());
+newParentTables.add(newParentTableIdentifier);
+  }
+  dataMapSchema.setParentTables(newParentTables);
+  newDataMapSchemas.add(dataMapSchema);
+  // frist drop old schema
+  String dataMapName = dataMapSchema.getDataMapName();
+  dropDataMapSchema(dataMapName);
+

Jenkins build is still unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Spark Common Test #2622

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Processing #2622

[Apache Jenkins Server]

See 

Jenkins build is still unstable: carbondata-master-spark-2.1 #2622

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.2 #664

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.2 » Apache CarbonData :: Spark Common Test #664

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.2 » Apache CarbonData :: Processing #664

[Apache Jenkins Server]

See 

Jenkins build is still unstable: carbondata-master-spark-2.2 #663

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-2.2 » Apache CarbonData :: Processing #663

[Apache Jenkins Server]

See 

Jenkins build is still unstable: carbondata-master-spark-2.2 » Apache CarbonData :: Spark Common Test #663

[Apache Jenkins Server]

See 

Jenkins build is still unstable: carbondata-master-spark-2.2 #662

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.2 » Apache CarbonData :: Processing #662

[Apache Jenkins Server]

See 

Jenkins build is still unstable: carbondata-master-spark-2.2 » Apache CarbonData :: Spark Common Test #662

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Spark Common Test #2621

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-2.1 #2621

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Spark2 #2623

[Apache Jenkins Server]

See 

carbondata git commit: [CARBONDATA-2606][Complex DataType Enhancements]Fix Null result if First two Projection column have same parent and third column has different Parent Struct

[kunalkapoor]

Repository: carbondata
Updated Branches:
  refs/heads/master 0c33857fd -> 75a602d01


[CARBONDATA-2606][Complex DataType Enhancements]Fix Null result if First two
 Projection column have same parent and third column has different Parent Struct

Problem:
When multiple columns are there,then the first child elements is only going
to make parent Object Array. For all other cases it should be null.
For e.g. a : . here as 'a' is the parent column and b, c, d are
child columns during traversal when we encounter the first element in
list i.e. column 'b','a' will be completely filled. In case when column
'c' and 'd' encountered then only place null in the output.
Hence, as Null is placed in the output, Select result is Null if First
two Projection column have same parent and third column has different parent 
Struct column.
Solution: Place null in the end of output

This closes #2489


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/75a602d0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/75a602d0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/75a602d0

Branch: refs/heads/master
Commit: 75a602d013fce170d59e66b4f907ff405f855d33
Parents: 0c33857
Author: Indhumathi27 
Authored: Wed Jul 11 15:54:55 2018 +0530
Committer: kunal642 
Committed: Sun Jul 15 12:54:28 2018 +0530

--
 .../impl/DictionaryBasedResultCollector.java| 11 +++
 .../core/scan/result/BlockletScannedResult.java |  2 +-
 .../complexType/TestComplexDataType.scala   | 31 
 .../sql/CarbonDatasourceHadoopRelation.scala|  2 +-
 4 files changed, 44 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/75a602d0/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index 495d7de..3184d80 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -140,6 +140,17 @@ public class DictionaryBasedResultCollector extends 
AbstractScannedResultCollect
 continue;
   }
   fillMeasureData(scannedResult, row);
+  if 
(scannedResult.complexParentIndexToQueryMap.toString().contains("StructQueryType"))
 {
+// If a :  and d :  are two struct and if a.b,a.c,d.e is 
given in the
+// projection list,then object array will contain a,null,d as result, 
because for a.b,
+// a will be filled and for a.c null will be placed.
+// Instead place null in the end of object array and send a,d,null as 
result.
+int count = 0;
+for (int j = 0; j < row.length; j++) {
+  if (row[j] != null) row[count++] = row[j];
+}
+while (count < row.length) row[count++] = null;
+  }
   listBasedResult.add(row);
   rowCounter++;
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75a602d0/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
index aac76e8..a25ebff 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
@@ -106,7 +106,7 @@ public abstract class BlockletScannedResult {
   /**
*
*/
-  private Map complexParentIndexToQueryMap;
+  public Map complexParentIndexToQueryMap;
 
   private int totalDimensionsSize;
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75a602d0/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
index 1068ba2..45a9c7a 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
+++