KYLIN-1179 Cannot use String as partition column Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/225a7d5b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/225a7d5b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/225a7d5b Branch: refs/heads/2.x-staging Commit: 225a7d5b6c68e4c0d5a70283823e4fede59d5522 Parents: 9f60f47 Author: lidongsjtu <don...@ebay.com> Authored: Mon Nov 30 10:48:36 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Fri Dec 4 16:29:06 2015 +0800 ---------------------------------------------------------------------- .../hbase/cube/v2/CubeSegmentScanner.java | 26 +++++++++++--------- webapp/app/js/controllers/modelEdit.js | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/225a7d5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java index 1eed318..2f78a86 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Set; +import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.DateFormat; @@ -30,6 +31,7 @@ import org.apache.kylin.gridtable.GTScanRequest; import org.apache.kylin.gridtable.GTUtil; import org.apache.kylin.gridtable.IGTScanner; import org.apache.kylin.metadata.filter.TupleFilter; +import org.apache.kylin.metadata.model.DataType; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -83,27 +85,23 @@ public class CubeSegmentScanner implements IGTScanner { GTInfo trimmedInfo = GTInfo.deserialize(trimmedInfoBytes); for (GTScanRange range : scanRanges) { - scanRequests.add(new GTScanRequest(trimmedInfo, range.replaceGTInfo(trimmedInfo), - gtDimensions, gtAggrGroups, gtAggrMetrics, gtAggrFuncs, gtFilter, allowPreAggregate)); + scanRequests.add(new GTScanRequest(trimmedInfo, range.replaceGTInfo(trimmedInfo), gtDimensions, gtAggrGroups, gtAggrMetrics, gtAggrFuncs, gtFilter, allowPreAggregate)); } scanner = new Scanner(); } private Pair<ByteArray, ByteArray> getSegmentStartAndEnd(TblColRef tblColRef, int index) { - - String partitionColType = tblColRef.getColumnDesc().getDatatype(); - ByteArray start; if (cubeSeg.getDateRangeStart() != Long.MIN_VALUE) { - start = translateTsToString(cubeSeg.getDateRangeStart(), partitionColType, index); + start = translateTsToString(cubeSeg.getDateRangeStart(), index, 1); } else { start = new ByteArray(); } ByteArray end; if (cubeSeg.getDateRangeEnd() != Long.MAX_VALUE) { - end = translateTsToString(cubeSeg.getDateRangeEnd(), partitionColType, index); + end = translateTsToString(cubeSeg.getDateRangeEnd(), index, -1); } else { end = new ByteArray(); } @@ -111,19 +109,25 @@ public class CubeSegmentScanner implements IGTScanner { } - private ByteArray translateTsToString(long ts, String partitionColType, int index) { + private ByteArray translateTsToString(long ts, int index, int roundingFlag) { String value; - if ("date".equalsIgnoreCase(partitionColType)) { + DataType partitionColType = info.getColumnType(index); + if (partitionColType.isDate()) { value = DateFormat.formatToDateStr(ts); - } else if ("timestamp".equalsIgnoreCase(partitionColType)) { + } else if (partitionColType.isDatetime()) { //TODO: if partition col is not dict encoded, value's format may differ from expected. Though by default it is not the case value = DateFormat.formatToTimeWithoutMilliStr(ts); + } else if (partitionColType.isStringFamily()) { + String partitionDateFormat = cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateFormat(); + if (StringUtils.isEmpty(partitionDateFormat)) + partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN; + value = DateFormat.formatToDateStr(ts, partitionDateFormat); } else { throw new RuntimeException("Type " + partitionColType + " is not valid partition column type"); } ByteBuffer buffer = ByteBuffer.allocate(info.getMaxColumnLength()); - info.getCodeSystem().encodeColumnValue(index, value, buffer); + info.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer); return ByteArray.copyOf(buffer.array(), 0, buffer.position()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/225a7d5b/webapp/app/js/controllers/modelEdit.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js index cbfb57f..70d4159 100644 --- a/webapp/app/js/controllers/modelEdit.js +++ b/webapp/app/js/controllers/modelEdit.js @@ -35,7 +35,7 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati $scope.getPartitonColumns = function(tableName){ var columns = _.filter($scope.getColumnsByTable(tableName),function(column){ - return column.datatype==="date"||column.datatype==="string"||column.datatype==="timestamp"; + return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar"); }); return columns; };
