Repository: carbondata Updated Branches: refs/heads/master daa91c88e -> 4c692d185
[HOTFIX] Old stores cannot read with new table infered through sdk. Problem: Old stores column schema is written in the different case then fileformat cannot read data because of sdk infer schema always gives lower case schema. Solution: Do case insensitivity check while comparing. It also disables prefetch as it is redundant for fileformat read and not getting inputmetrics properly if we use thread This closes #2704 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4c692d18 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4c692d18 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4c692d18 Branch: refs/heads/master Commit: 4c692d185c4247e645d94c2d79787744c413817b Parents: daa91c8 Author: ravipesala <[email protected]> Authored: Mon Sep 10 21:11:18 2018 +0530 Committer: kumarvishal09 <[email protected]> Committed: Wed Sep 12 19:16:39 2018 +0530 ---------------------------------------------------------------------- .../apache/carbondata/core/metadata/CarbonMetadata.java | 5 +++-- .../metadata/schema/table/AggregationDataMapSchema.java | 4 ++-- .../core/metadata/schema/table/column/ColumnSchema.java | 2 +- .../core/scan/executor/impl/AbstractQueryExecutor.java | 6 +++++- .../core/scan/executor/util/RestructureUtil.java | 7 ++++--- .../scan/expression/logical/BinaryLogicalExpression.java | 2 +- .../apache/carbondata/core/scan/filter/FilterUtil.java | 2 +- .../org/apache/carbondata/core/scan/model/QueryModel.java | 10 ++++++++++ .../apache/carbondata/core/util/BlockletDataMapUtil.java | 2 +- .../java/org/apache/carbondata/core/util/CarbonUtil.java | 2 +- .../execution/datasources/SparkCarbonFileFormat.scala | 1 + 11 files changed, 30 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java index 3f8c12d..850f477 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java @@ -143,7 +143,7 @@ public final class CarbonMetadata { List<CarbonDimension> listOfCarbonDims = carbonTable.getDimensionByTableName(carbonTable.getTableName()); for (CarbonDimension dimension : listOfCarbonDims) { - if (dimension.getColumnId().equals(columnIdentifier)) { + if (dimension.getColumnId().equalsIgnoreCase(columnIdentifier)) { return dimension; } if (dimension.getNumberOfChild() > 0) { @@ -168,7 +168,8 @@ public final class CarbonMetadata { private CarbonDimension getCarbonChildDimsBasedOnColIdentifier(String columnIdentifier, CarbonDimension dimension) { for (int i = 0; i < dimension.getNumberOfChild(); i++) { - if (dimension.getListOfChildDimensions().get(i).getColumnId().equals(columnIdentifier)) { + if (dimension.getListOfChildDimensions().get(i).getColumnId() + .equalsIgnoreCase(columnIdentifier)) { return dimension.getListOfChildDimensions().get(i); } else if (dimension.getListOfChildDimensions().get(i).getNumberOfChild() > 0) { CarbonDimension childDim = getCarbonChildDimsBasedOnColIdentifier(columnIdentifier, http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java index 2bb6d18..c8bb5ad 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java @@ -152,7 +152,7 @@ public class AggregationDataMapSchema extends DataMapSchema { List<ParentColumnTableRelation> parentColumnTableRelations = columnSchema.getParentColumnTableRelations(); if (null != parentColumnTableRelations && parentColumnTableRelations.size() == 1 - && parentColumnTableRelations.get(0).getColumnName().equals(columName) && + && parentColumnTableRelations.get(0).getColumnName().equalsIgnoreCase(columName) && columnSchema.getColumnName().endsWith(columName)) { return columnSchema; } @@ -198,7 +198,7 @@ public class AggregationDataMapSchema extends DataMapSchema { List<ParentColumnTableRelation> parentColumnTableRelations = columnSchema.getParentColumnTableRelations(); if (null != parentColumnTableRelations && parentColumnTableRelations.size() == 1 - && parentColumnTableRelations.get(0).getColumnName().equals(columName) + && parentColumnTableRelations.get(0).getColumnName().equalsIgnoreCase(columName) && timeseriesFunction.equalsIgnoreCase(columnSchema.getTimeSeriesFunction())) { return columnSchema; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java index 853851d..8eba101 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java @@ -315,7 +315,7 @@ public class ColumnSchema implements Serializable, Writable { if (other.columnName != null) { return false; } - } else if (!columnName.equals(other.columnName)) { + } else if (!columnName.equalsIgnoreCase(other.columnName)) { return false; } if (dataType == null) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java index bd1eb1c..83276e0 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java @@ -476,7 +476,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { blockExecutionInfo .setTotalNumberDimensionToRead( segmentProperties.getDimensionOrdinalToChunkMapping().size()); - blockExecutionInfo.setPrefetchBlocklet(!queryModel.isReadPageByPage()); + if (queryModel.isReadPageByPage()) { + blockExecutionInfo.setPrefetchBlocklet(false); + } else { + blockExecutionInfo.setPrefetchBlocklet(queryModel.isPreFetchData()); + } blockExecutionInfo .setTotalNumberOfMeasureToRead(segmentProperties.getMeasuresOrdinalToChunkMapping().size()); blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java index 9fe2a38..e823eb2 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java @@ -165,14 +165,15 @@ public class RestructureUtil { // column ID but can have same column name if (tableColumn.getDataType().isComplexType() && !(tableColumn.getDataType().getId() == DataTypes.ARRAY_TYPE_ID)) { - if (tableColumn.getColumnId().equals(queryColumn.getColumnId())) { + if (tableColumn.getColumnId().equalsIgnoreCase(queryColumn.getColumnId())) { return true; } else { return isColumnMatchesStruct(tableColumn, queryColumn); } } else { - return (tableColumn.getColumnId().equals(queryColumn.getColumnId()) || (!isTransactionalTable - && tableColumn.getColName().equals(queryColumn.getColName()))); + return (tableColumn.getColumnId().equalsIgnoreCase(queryColumn.getColumnId()) || ( + !isTransactionalTable && tableColumn.getColName() + .equalsIgnoreCase(queryColumn.getColName()))); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java index 1a5b6a7..ecbb8d9 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java @@ -64,7 +64,7 @@ public abstract class BinaryLogicalExpression extends BinaryExpression { boolean found = false; for (ColumnExpression currentColExp : lst) { - if (currentColExp.getColumnName().equals(colExp.getColumnName())) { + if (currentColExp.getColumnName().equalsIgnoreCase(colExp.getColumnName())) { found = true; colExp.setColIndex(currentColExp.getColIndex()); break; http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index bae608f..410f67a 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -397,7 +397,7 @@ public final class FilterUtil { int columnIndexInMinMaxByteArray = -1; int columnCounter = 0; for (CarbonColumn cachedColumn : carbonDimensionsToBeCached) { - if (cachedColumn.getColumnId().equals(filterColumn.getColumnId())) { + if (cachedColumn.getColumnId().equalsIgnoreCase(filterColumn.getColumnId())) { columnIndexInMinMaxByteArray = columnCounter; break; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java index 6df98e6..d90c35e 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java @@ -122,6 +122,8 @@ public class QueryModel { // whether to clear/free unsafe memory or not private boolean freeUnsafeMemory = true; + private boolean preFetchData = true; + private QueryModel(CarbonTable carbonTable) { tableBlockInfos = new ArrayList<TableBlockInfo>(); invalidSegmentIds = new ArrayList<>(); @@ -396,6 +398,14 @@ public class QueryModel { isFG = FG; } + public boolean isPreFetchData() { + return preFetchData; + } + + public void setPreFetchData(boolean preFetchData) { + this.preFetchData = preFetchData; + } + @Override public String toString() { return String.format("scan on table %s.%s, %d projection columns with filter (%s)", http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java b/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java index 1832cf5..4a87e91 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java @@ -479,7 +479,7 @@ public class BlockletDataMapUtil { private static boolean filterColumnExistsInMinMaxColumnList(List<CarbonColumn> minMaxCacheColumns, CarbonColumn filterColumn) { for (CarbonColumn column : minMaxCacheColumns) { - if (filterColumn.getColumnId().equals(column.getColumnId())) { + if (filterColumn.getColumnId().equalsIgnoreCase(column.getColumnId())) { return true; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index d71f294..b1e3ef0 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -1193,7 +1193,7 @@ public final class CarbonUtil { List<CarbonDimension> blockDimensions, CarbonDimension dimensionToBeSearched) { CarbonDimension currentBlockDimension = null; for (CarbonDimension blockDimension : blockDimensions) { - if (dimensionToBeSearched.getColumnId().equals(blockDimension.getColumnId())) { + if (dimensionToBeSearched.getColumnId().equalsIgnoreCase(blockDimension.getColumnId())) { currentBlockDimension = blockDimension; break; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala ---------------------------------------------------------------------- diff --git a/integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala b/integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala index 62d9903..a6965ac 100644 --- a/integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala +++ b/integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala @@ -387,6 +387,7 @@ class SparkCarbonFileFormat extends FileFormat new TaskAttemptContextImpl(broadcastedHadoopConf.value.value, attemptId) val model = format.createQueryModel(split, hadoopAttemptContext) model.setConverter(new SparkDataTypeConverterImpl) + model.setPreFetchData(false) val carbonReader = if (readVector) { val vectorizedReader = new VectorizedCarbonRecordReader(model, null,
