This is an automated email from the ASF dual-hosted git repository.
akashrn5 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 6ddf1b9 [CARBONDATA-3990] Fix DropCache log error when indexmap is
null on SI table
6ddf1b9 is described below
commit 6ddf1b904d5437991e084a8ddfec6a608de1edb6
Author: Indhumathi27 <[email protected]>
AuthorDate: Tue Sep 15 12:09:04 2020 +0530
[CARBONDATA-3990] Fix DropCache log error when indexmap is null on SI table
Why is this PR needed?
1. DropCache on Maintable having SI throws NPE in error log, when
refreshRequired
for SI table Schema is false.
2. While direct upgrading store to current version for tables having SI,
query
fails with NPE, as indexProperties is null.
What changes were proposed in this PR?
1. Check if indexMetadata has indexmap or not, in case of index table, this
value
will be null or empty. if null, return empty list.
2. If indexProperties is null in case of compatibility scenarios, prepare
indexProperties from indexColumns
3. Removed throwable in warning message on listFiles
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3928
---
.../filesystem/AbstractDFSCarbonFile.java | 2 +-
.../carbondata/core/index/IndexStoreManager.java | 26 +++++++++-------------
.../core/metadata/schema/table/CarbonTable.java | 2 +-
.../hive/CarbonInternalMetastore.scala | 25 ++++++++++++++-------
4 files changed, 30 insertions(+), 25 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index 9278421..338076e 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -457,7 +457,7 @@ public abstract class AbstractDFSCarbonFile implements
CarbonFile {
try {
listStatus = fileSystem.listStatus(path);
} catch (IOException e) {
- LOGGER.warn("Exception occurred: " + e.getMessage(), e);
+ LOGGER.warn("Exception occurred: " + e.getMessage());
return new CarbonFile[0];
}
return getFiles(listStatus);
diff --git
a/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
b/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
index e96889a..8d4be8f 100644
--- a/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
@@ -38,7 +38,6 @@ import
org.apache.carbondata.core.indexstore.blockletindex.BlockletIndexFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.index.IndexType;
-import org.apache.carbondata.core.metadata.schema.indextable.IndexMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.IndexSchema;
import org.apache.carbondata.core.mutate.UpdateVO;
@@ -90,21 +89,18 @@ public final class IndexStoreManager {
* @return
*/
public List<TableIndex> getAllCGAndFGIndexes(CarbonTable carbonTable) throws
IOException {
- IndexMetadata indexMetadata = carbonTable.getIndexMetadata();
List<TableIndex> indexes = new ArrayList<>();
- if (null != indexMetadata) {
- // get bloom indexes and lucene indexes
- for (Map.Entry<String, Map<String, Map<String, String>>> providerEntry :
indexMetadata
- .getIndexesMap().entrySet()) {
- for (Map.Entry<String, Map<String, String>> indexEntry :
providerEntry.getValue()
- .entrySet()) {
- if (!indexEntry.getValue().get(CarbonCommonConstants.INDEX_PROVIDER)
- .equalsIgnoreCase(IndexType.SI.getIndexProviderName())) {
- IndexSchema indexSchema = new IndexSchema(indexEntry.getKey(),
-
indexEntry.getValue().get(CarbonCommonConstants.INDEX_PROVIDER));
- indexSchema.setProperties(indexEntry.getValue());
- indexes.add(getIndex(carbonTable, indexSchema));
- }
+ // get bloom indexes and lucene indexes
+ for (Map.Entry<String, Map<String, Map<String, String>>> providerEntry :
carbonTable
+ .getIndexesMap().entrySet()) {
+ for (Map.Entry<String, Map<String, String>> indexEntry :
providerEntry.getValue()
+ .entrySet()) {
+ if (!indexEntry.getValue().get(CarbonCommonConstants.INDEX_PROVIDER)
+ .equalsIgnoreCase(IndexType.SI.getIndexProviderName())) {
+ IndexSchema indexSchema = new IndexSchema(indexEntry.getKey(),
+ indexEntry.getValue().get(CarbonCommonConstants.INDEX_PROVIDER));
+ indexSchema.setProperties(indexEntry.getValue());
+ indexes.add(getIndex(carbonTable, indexSchema));
}
}
}
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index cbaf18a..4fd58cd 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -1176,7 +1176,7 @@ public class CarbonTable implements Serializable,
Writable {
public Map<String, Map<String, Map<String, String>>> getIndexesMap() throws
IOException {
deserializeIndexMetadata();
- if (null == indexMetadata) {
+ if (null == indexMetadata || null == indexMetadata.getIndexesMap()) {
return new HashMap<>();
}
return indexMetadata.getIndexesMap();
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/hive/CarbonInternalMetastore.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/hive/CarbonInternalMetastore.scala
index 9049bbd..539c5f2 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/hive/CarbonInternalMetastore.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/hive/CarbonInternalMetastore.scala
@@ -167,21 +167,30 @@ object CarbonInternalMetastore {
} else {
IndexTableInfo.fromGson(indexInfo)
.foreach { indexTableInfo =>
- val indexProvider = indexTableInfo.getIndexProperties
- .get(CarbonCommonConstants.INDEX_PROVIDER)
+ var indexProperties = indexTableInfo.getIndexProperties
+ val indexProvider = if (null != indexProperties) {
+ indexProperties.get(CarbonCommonConstants.INDEX_PROVIDER)
+ } else {
+ // in case if SI table has been created before the change
CARBONDATA-3765,
+ // indexProperties variable will not be present. On direct
upgrade of SI store,
+ // indexProperties will be null, in that case, create
indexProperties from indexCols
+ indexProperties = new java.util.HashMap[String, String]()
+ indexProperties.put(CarbonCommonConstants.INDEX_COLUMNS,
+ indexTableInfo.getIndexCols.asScala.mkString(","))
+ val provider = IndexType.SI.getIndexProviderName
+ indexProperties.put(CarbonCommonConstants.INDEX_PROVIDER,
provider)
+ provider
+ }
if (indexTableMap.isEmpty) {
val indexTableInfoMap = new java.util.HashMap[String,
java.util.Map[String, String]]
- indexTableInfoMap.put(indexTableInfo.getTableName,
- indexTableInfo.getIndexProperties)
+ indexTableInfoMap.put(indexTableInfo.getTableName,
indexProperties)
indexTableMap.put(indexProvider, indexTableInfoMap)
} else if (null == indexTableMap.get(indexProvider)) {
val indexTableInfoMap = new java.util.HashMap[String,
java.util.Map[String, String]]
- indexTableInfoMap.put(indexTableInfo.getTableName,
- indexTableInfo.getIndexProperties)
+ indexTableInfoMap.put(indexTableInfo.getTableName,
indexProperties)
indexTableMap.put(indexProvider, indexTableInfoMap)
} else {
-
indexTableMap.get(indexProvider).put(indexTableInfo.getTableName,
- indexTableInfo.getIndexProperties)
+
indexTableMap.get(indexProvider).put(indexTableInfo.getTableName,
indexProperties)
}
}
val indexMetadata = new IndexMetadata(
