Repository: carbondata Updated Branches: refs/heads/master b3f782062 -> 18dc3ff49
[CARBONDATA-2626] Fix bugs in sequence of column page In tablespec, the first dimension spec is 'name' which is sort_columns but not dictionary. But Carbondata now treat it as dictionary page. (See TablePage Line94). This PR fixes this bug This closes #2393 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/18dc3ff4 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/18dc3ff4 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/18dc3ff4 Branch: refs/heads/master Commit: 18dc3ff49b23d071a545e4645f4565a45274c587 Parents: b3f7820 Author: xuchuanyin <[email protected]> Authored: Thu Jun 21 17:51:30 2018 +0800 Committer: Jacky Li <[email protected]> Committed: Fri Jun 22 15:17:09 2018 +0800 ---------------------------------------------------------------------- .../carbondata/processing/store/TablePage.java | 38 +++++++++++--------- 1 file changed, 22 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/18dc3ff4/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java index 1a1c5d1..b1b966b 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java @@ -89,28 +89,34 @@ public class TablePage { this.pageSize = pageSize; int numDictDimension = model.getMDKeyGenerator().getDimCount(); TableSpec tableSpec = model.getTableSpec(); + dictDimensionPages = new ColumnPage[numDictDimension]; - for (int i = 0; i < dictDimensionPages.length; i++) { - TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i); - ColumnPage page = ColumnPage.newPage(spec, DataTypes.BYTE_ARRAY, pageSize); - page.setStatsCollector(KeyPageStatsCollector.newInstance(DataTypes.BYTE_ARRAY)); - dictDimensionPages[i] = page; - } noDictDimensionPages = new ColumnPage[model.getNoDictionaryCount()]; - for (int i = 0; i < noDictDimensionPages.length; i++) { - TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i + numDictDimension); + int tmpNumDictDimIdx = 0; + int tmpNumNoDictDimIdx = 0; + for (int i = 0; i < dictDimensionPages.length + noDictDimensionPages.length; i++) { + TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i); + ColumnType columnType = tableSpec.getDimensionSpec(i).getColumnType(); ColumnPage page; - if (DataTypes.VARCHAR == spec.getSchemaDataType()) { - page = ColumnPage.newPage(spec, DataTypes.VARCHAR, pageSize); - page.setStatsCollector(LVLongStringStatsCollector.newInstance()); + if (ColumnType.GLOBAL_DICTIONARY == columnType + || ColumnType.DIRECT_DICTIONARY == columnType) { + page = ColumnPage.newPage(spec, DataTypes.BYTE_ARRAY, pageSize); + page.setStatsCollector(KeyPageStatsCollector.newInstance(DataTypes.BYTE_ARRAY)); + dictDimensionPages[tmpNumDictDimIdx++] = page; } else { - // In previous implementation, other data types such as string, date and timestamp - // will be encoded using string page - page = ColumnPage.newPage(spec, DataTypes.STRING, pageSize); - page.setStatsCollector(LVShortStringStatsCollector.newInstance()); + if (DataTypes.VARCHAR == spec.getSchemaDataType()) { + page = ColumnPage.newPage(spec, DataTypes.VARCHAR, pageSize); + page.setStatsCollector(LVLongStringStatsCollector.newInstance()); + } else { + // In previous implementation, other data types such as string, date and timestamp + // will be encoded using string page + page = ColumnPage.newPage(spec, DataTypes.STRING, pageSize); + page.setStatsCollector(LVShortStringStatsCollector.newInstance()); + } + noDictDimensionPages[tmpNumNoDictDimIdx++] = page; } - noDictDimensionPages[i] = page; } + complexDimensionPages = new ComplexColumnPage[model.getComplexColumnCount()]; for (int i = 0; i < complexDimensionPages.length; i++) { // here we still do not the depth of the complex column, it will be initialized when
