[4/6] carbondata git commit: [CARBONDATA-2244]fix creating pre-aggregate table bug when there are invisibility INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main table

2018-03-16 Thread jackylk 
[CARBONDATA-2244]fix creating pre-aggregate table bug when there are 
invisibility INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main 
table

When there are some invisibility 
INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main table, it can 
not create preaggregate table on it.

This closes #2050


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a386f1f4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a386f1f4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a386f1f4

Branch: refs/heads/carbonfile
Commit: a386f1f4e220210e031f10a73c26a7f56f57a603
Parents: 31011fc
Author: Zhang Zhichao <441586...@qq.com>
Authored: Fri Mar 9 23:36:08 2018 +0800
Committer: Jacky Li 
Committed: Fri Mar 16 14:43:06 2018 +0800

--
 .../command/preaaggregate/PreAggregateTableHelper.scala  | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/a386f1f4/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
index 1f1e1e6..b64c91e 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
@@ -165,12 +165,12 @@ case class PreAggregateTableHelper(
 // need to fire load for pre-aggregate table. Therefore reading the load 
details for PARENT
 // table.
 SegmentStatusManager.deleteLoadsAndUpdateMetadata(parentTable, false)
-val loadAvailable = 
SegmentStatusManager.readLoadMetadata(parentTable.getMetadataPath)
-if (loadAvailable.exists(load => load.getSegmentStatus == 
SegmentStatus.INSERT_IN_PROGRESS ||
-  load.getSegmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)) {
+if (SegmentStatusManager.isLoadInProgressInTable(parentTable)) {
   throw new UnsupportedOperationException(
 "Cannot create pre-aggregate table when insert is in progress on main 
table")
-} else if (loadAvailable.nonEmpty) {
+}
+val loadAvailable = 
SegmentStatusManager.readLoadMetadata(parentTable.getMetadataPath)
+if (loadAvailable.nonEmpty) {
   // Passing segmentToLoad as * because we want to load all the segments 
into the
   // pre-aggregate table even if the user has set some segments on the 
parent table.
   loadCommand.dataFrame = Some(PreAggregateUtil

carbondata git commit: [CARBONDATA-2244]fix creating pre-aggregate table bug when there are invisibility INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main table

2018-03-16 Thread jackylk 
Repository: carbondata
Updated Branches:
  refs/heads/branch-1.3 0283c938b -> 96e26d7a5


[CARBONDATA-2244]fix creating pre-aggregate table bug when there are 
invisibility INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main 
table

When there are some invisibility 
INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main table, it can 
not create preaggregate table on it.

This closes #2050


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/96e26d7a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/96e26d7a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/96e26d7a

Branch: refs/heads/branch-1.3
Commit: 96e26d7a5da347c3638827f5986aebc7b1eaa237
Parents: 0283c93
Author: Zhang Zhichao <441586...@qq.com>
Authored: Fri Mar 9 23:36:08 2018 +0800
Committer: Jacky Li 
Committed: Fri Mar 16 14:47:22 2018 +0800

--
 .../CreatePreAggregateTableCommand.scala| 23 
 1 file changed, 14 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/96e26d7a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index 59c43aa..4270d47 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -179,17 +179,22 @@ case class CreatePreAggregateTableCommand(
 // This will be used to check if the parent table has any segments or not. 
If not then no
 // need to fire load for pre-aggregate table. Therefore reading the load 
details for PARENT
 // table.
-DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false,
+DataLoadingUtil.deleteLoadsAndUpdateMetadata(
+  isForceDeletion = false,
   parentTable,
-  CarbonFilters.getCurrentPartitions(sparkSession,
-  TableIdentifier(parentTable.getTableName,
-Some(parentTable.getDatabaseName))).map(_.asJava).orNull)
-val loadAvailable = 
SegmentStatusManager.readLoadMetadata(parentTable.getMetaDataFilepath)
-if (loadAvailable.exists(load => load.getSegmentStatus == 
SegmentStatus.INSERT_IN_PROGRESS ||
-  load.getSegmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)) {
+  CarbonFilters.getCurrentPartitions(
+sparkSession,
+TableIdentifier(parentTable.getTableName,
+Some(parentTable.getDatabaseName))
+  ).map(_.asJava).orNull
+)
+
+if (SegmentStatusManager.isLoadInProgressInTable(parentTable)) {
   throw new UnsupportedOperationException(
-"Cannot create pre-aggregate table when insert is in progress on main 
table")
-} else if (loadAvailable.nonEmpty) {
+  "Cannot create pre-aggregate table when insert is in progress on main 
table")
+}
+val loadAvailable = 
SegmentStatusManager.readLoadMetadata(parentTable.getMetaDataFilepath)
+if (loadAvailable.nonEmpty) {
   // Passing segmentToLoad as * because we want to load all the segments 
into the
   // pre-aggregate table even if the user has set some segments on the 
parent table.
   loadCommand.dataFrame = Some(PreAggregateUtil

carbondata git commit: [CARBONDATA-2244]fix creating pre-aggregate table bug when there are invisibility INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main table

2018-03-16 Thread jackylk 
Repository: carbondata
Updated Branches:
  refs/heads/master 31011fc29 -> a386f1f4e


[CARBONDATA-2244]fix creating pre-aggregate table bug when there are 
invisibility INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main 
table

When there are some invisibility 
INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main table, it can 
not create preaggregate table on it.

This closes #2050


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a386f1f4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a386f1f4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a386f1f4

Branch: refs/heads/master
Commit: a386f1f4e220210e031f10a73c26a7f56f57a603
Parents: 31011fc
Author: Zhang Zhichao <441586...@qq.com>
Authored: Fri Mar 9 23:36:08 2018 +0800
Committer: Jacky Li 
Committed: Fri Mar 16 14:43:06 2018 +0800

--
 .../command/preaaggregate/PreAggregateTableHelper.scala  | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/a386f1f4/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
index 1f1e1e6..b64c91e 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
@@ -165,12 +165,12 @@ case class PreAggregateTableHelper(
 // need to fire load for pre-aggregate table. Therefore reading the load 
details for PARENT
 // table.
 SegmentStatusManager.deleteLoadsAndUpdateMetadata(parentTable, false)
-val loadAvailable = 
SegmentStatusManager.readLoadMetadata(parentTable.getMetadataPath)
-if (loadAvailable.exists(load => load.getSegmentStatus == 
SegmentStatus.INSERT_IN_PROGRESS ||
-  load.getSegmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)) {
+if (SegmentStatusManager.isLoadInProgressInTable(parentTable)) {
   throw new UnsupportedOperationException(
 "Cannot create pre-aggregate table when insert is in progress on main 
table")
-} else if (loadAvailable.nonEmpty) {
+}
+val loadAvailable = 
SegmentStatusManager.readLoadMetadata(parentTable.getMetadataPath)
+if (loadAvailable.nonEmpty) {
   // Passing segmentToLoad as * because we want to load all the segments 
into the
   // pre-aggregate table even if the user has set some segments on the 
parent table.
   loadCommand.dataFrame = Some(PreAggregateUtil