carbondata git commit: [CARBONDATA-2142] [CARBONDATA-1763] Fixed issues while creation concurrent datamaps
Repository: carbondata
Updated Branches:
refs/heads/branch-1.3 7beef112b -> 39ac94e46
[CARBONDATA-2142] [CARBONDATA-1763] Fixed issues while creation concurrent
datamaps
Analysis:
1. GenerateTableSchemaString in CarbonMetastore did not have any specific
implementation for hive metastore due to which carbontables were being
cached in MetaData. As there is no way to refresh table in hivemetastore
therefore this is wrong. All queries should get the latest carbon table
from metastore and not from cache.
2. If updating the main table status fails then revertMainTableChanges method
is called to revert the changes. The logic to revert was wrong which led
to wrong entry getting deleted from the schema.
3. Moved the force remove logic before taking locks as deletion from metastore
should happen even if the lock if not present as the table is in
stale state(Entry is not there in parent but available in metastore).
This closes #1975
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/39ac94e4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/39ac94e4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/39ac94e4
Branch: refs/heads/branch-1.3
Commit: 39ac94e462e6571414dee8f58c174e44a79f8ad4
Parents: 7beef11
Author: kunal642
Authored: Tue Feb 13 00:53:31 2018 +0530
Committer: manishgupta88
Committed: Sat Feb 24 11:01:45 2018 +0530
--
.../carbondata/core/locks/CarbonLockUtil.java | 6 ++-
.../core/metadata/AbsoluteTableIdentifier.java | 5 +++
.../datamap/CarbonDropDataMapCommand.scala | 46
.../CreatePreAggregateTableCommand.scala| 16 ---
.../preaaggregate/PreAggregateUtil.scala| 27 +---
.../spark/sql/hive/CarbonHiveMetaStore.scala| 14 +-
6 files changed, 64 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/39ac94e4/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
index 1fcccfb..c399ef4 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
@@ -74,9 +74,11 @@ public class CarbonLockUtil {
public static ICarbonLock getLockObject(AbsoluteTableIdentifier
absoluteTableIdentifier,
String lockType, String errorMsg) {
ICarbonLock carbonLock =
CarbonLockFactory.getCarbonLockObj(absoluteTableIdentifier, lockType);
-LOGGER.info("Trying to acquire lock: " + carbonLock);
+LOGGER.info("Trying to acquire lock: " + lockType + "for table: " +
+absoluteTableIdentifier.toString());
if (carbonLock.lockWithRetries()) {
- LOGGER.info("Successfully acquired the lock " + carbonLock);
+ LOGGER.info("Successfully acquired the lock " + lockType + "for table: "
+
+ absoluteTableIdentifier.toString());
} else {
LOGGER.error(errorMsg);
throw new RuntimeException(errorMsg);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/39ac94e4/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
index 6ef2671..d3250aa 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
@@ -143,4 +143,9 @@ public class AbsoluteTableIdentifier implements
Serializable {
public String getTableName() {
return carbonTableIdentifier.getTableName();
}
+
+ public String toString() {
+return carbonTableIdentifier.toString();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/39ac94e4/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
--
diff --git
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index 8ef394c..2675036 100644
---
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataM
carbondata git commit: [CARBONDATA-2142] [CARBONDATA-1763] Fixed issues while creation concurrent datamaps
Repository: carbondata
Updated Branches:
refs/heads/master 310b06de1 -> e9430312d
[CARBONDATA-2142] [CARBONDATA-1763] Fixed issues while creation concurrent
datamaps
Analysis:
1. GenerateTableSchemaString in CarbonMetastore did not have any specific
implementation for hive metastore due to which carbontables were being
cached in MetaData. As there is no way to refresh table in hivemetastore
therefore this is wrong. All queries should get the latest carbon table
from metastore and not from cache.
2. If updating the main table status fails then revertMainTableChanges method
is called to revert the changes. The logic to revert was wrong which led
to wrong entry getting deleted from the schema.
3. Moved the force remove logic before taking locks as deletion from metastore
should happen even if the lock if not present as the table is in
stale state(Entry is not there in parent but available in metastore).
This closes #1975
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e9430312
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e9430312
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e9430312
Branch: refs/heads/master
Commit: e9430312d61c0770f402e3475178addba3c37e5e
Parents: 310b06d
Author: kunal642
Authored: Tue Feb 13 00:53:31 2018 +0530
Committer: manishgupta88
Committed: Sat Feb 24 10:59:14 2018 +0530
--
.../carbondata/core/locks/CarbonLockUtil.java | 6 ++-
.../core/metadata/AbsoluteTableIdentifier.java | 5 +++
.../datamap/CarbonDropDataMapCommand.scala | 46
.../CreatePreAggregateTableCommand.scala| 16 ---
.../preaaggregate/PreAggregateUtil.scala| 27 +---
.../spark/sql/hive/CarbonHiveMetaStore.scala| 14 +-
6 files changed, 64 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9430312/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
index 1fcccfb..c399ef4 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
@@ -74,9 +74,11 @@ public class CarbonLockUtil {
public static ICarbonLock getLockObject(AbsoluteTableIdentifier
absoluteTableIdentifier,
String lockType, String errorMsg) {
ICarbonLock carbonLock =
CarbonLockFactory.getCarbonLockObj(absoluteTableIdentifier, lockType);
-LOGGER.info("Trying to acquire lock: " + carbonLock);
+LOGGER.info("Trying to acquire lock: " + lockType + "for table: " +
+absoluteTableIdentifier.toString());
if (carbonLock.lockWithRetries()) {
- LOGGER.info("Successfully acquired the lock " + carbonLock);
+ LOGGER.info("Successfully acquired the lock " + lockType + "for table: "
+
+ absoluteTableIdentifier.toString());
} else {
LOGGER.error(errorMsg);
throw new RuntimeException(errorMsg);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9430312/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
index 6ef2671..d3250aa 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
@@ -143,4 +143,9 @@ public class AbsoluteTableIdentifier implements
Serializable {
public String getTableName() {
return carbonTableIdentifier.getTableName();
}
+
+ public String toString() {
+return carbonTableIdentifier.toString();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9430312/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
--
diff --git
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index 8ef394c..2675036 100644
---
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapComman
