Repository: carbondata Updated Branches: refs/heads/master edfcdca0a -> f962e41b7
[CARBONDATA-2953]fixed dataload failure with sort columns and query wrong result from other session Problem: when dataload is done with sort columns, it fails with following exeptions when two sessions are running in parallel, the follow below steps in session1 drop table create table load data to table follow below step in session2 query on table(select * from table limit 1), then the query returns null result instead of proper result Solution During sorting, the index increament for no dictionary measure data was not happening correctly, hence was trying to cast to byte array and failing If table is dropped from first session and created again, and queries from another session, the metastore needs to be updated for newly created table, but since the database in identifier was None. we were trying to get old table from default database, here need to get from current database This closes #2743 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f962e41b Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f962e41b Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f962e41b Branch: refs/heads/master Commit: f962e41b7f2c2dd29ae71ad5e1f7797e3aaec084 Parents: edfcdca Author: akashrn5 <[email protected]> Authored: Thu Sep 20 15:39:01 2018 +0530 Committer: kumarvishal09 <[email protected]> Committed: Fri Sep 21 18:46:25 2018 +0530 ---------------------------------------------------------------------- .../execution/command/datamap/CarbonDataMapShowCommand.scala | 2 +- .../scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala | 5 ++++- .../processing/loading/partition/impl/RawRowComparator.java | 2 +- .../sort/sortdata/IntermediateSortTempRowComparator.java | 2 +- .../carbondata/processing/sort/sortdata/NewRowComparator.java | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/f962e41b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala index b583a30..ae33aa8 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala @@ -57,8 +57,8 @@ case class CarbonDataMapShowCommand(tableIdentifier: Option[TableIdentifier]) val dataMapSchemaList: util.List[DataMapSchema] = new util.ArrayList[DataMapSchema]() tableIdentifier match { case Some(table) => - Checker.validateTableExists(table.database, table.table, sparkSession) val carbonTable = CarbonEnv.getCarbonTable(table)(sparkSession) + Checker.validateTableExists(table.database, table.table, sparkSession) if (carbonTable.hasDataMapSchema) { dataMapSchemaList.addAll(carbonTable.getTableInfo.getDataMapSchemaList) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f962e41b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala index 1840c5d..982bbee 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala @@ -580,7 +580,10 @@ class CarbonFileMetastore extends CarbonMetaStore { tableModifiedTimeStore.get(CarbonCommonConstants.DATABASE_DEFAULT_NAME))) { metadata.carbonTables = metadata.carbonTables.filterNot( table => table.getTableName.equalsIgnoreCase(tableIdentifier.table) && - table.getDatabaseName.equalsIgnoreCase(tableIdentifier.database.getOrElse("default"))) + table.getDatabaseName + .equalsIgnoreCase(tableIdentifier.database + .getOrElse(SparkSession.getActiveSession.get.sessionState.catalog + .getCurrentDatabase))) updateSchemasUpdatedTime(lastModifiedTime) isRefreshed = true } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f962e41b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RawRowComparator.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RawRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RawRowComparator.java index 3a325a4..3c16b67 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RawRowComparator.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RawRowComparator.java @@ -57,7 +57,6 @@ public class RawRowComparator implements Comparator<CarbonRow> { if (difference != 0) { return difference; } - noDicIdx++; } else { byte[] colA = (byte[]) o1.getObject(colIdx); byte[] colB = (byte[]) o2.getObject(colIdx); @@ -66,6 +65,7 @@ public class RawRowComparator implements Comparator<CarbonRow> { return diff; } } + noDicIdx++; } else { int colA = (int) o1.getObject(colIdx); int colB = (int) o2.getObject(colIdx); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f962e41b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java index 54fa99e..62d541f 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java +++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java @@ -66,7 +66,6 @@ public class IntermediateSortTempRowComparator implements Comparator<Intermediat if (difference != 0) { return difference; } - noDicTypeIdx++; } else { byte[] byteArr1 = (byte[]) rowA.getNoDictSortDims()[nonDictIndex]; byte[] byteArr2 = (byte[]) rowB.getNoDictSortDims()[nonDictIndex]; @@ -77,6 +76,7 @@ public class IntermediateSortTempRowComparator implements Comparator<Intermediat } } nonDictIndex++; + noDicTypeIdx++; } else { int dimFieldA = rowA.getDictSortDims()[dictIndex]; int dimFieldB = rowB.getDictSortDims()[dictIndex]; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f962e41b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java index 4dff644..f213764 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java +++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java @@ -68,7 +68,6 @@ public class NewRowComparator implements Comparator<Object[]>, Serializable { if (difference != 0) { return difference; } - dataTypeIdx++; } else { byte[] byteArr1 = (byte[]) rowA[index]; byte[] byteArr2 = (byte[]) rowB[index]; @@ -79,6 +78,7 @@ public class NewRowComparator implements Comparator<Object[]>, Serializable { } } } + dataTypeIdx++; } else { int dimFieldA = (int) rowA[index]; int dimFieldB = (int) rowB[index];
