Repository: kylin Updated Branches: refs/heads/2.x-staging 7dc56a343 -> 6678ae372
KYLIN-1427 Support additional partition_time_column and partition_time_format besides partition_date_column and partition_date_format Signed-off-by: Li, Yang <[email protected]> This closes #27 Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6678ae37 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6678ae37 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6678ae37 Branch: refs/heads/2.x-staging Commit: 6678ae3725d030d4acb08e2aa5f20261aa7bf16b Parents: 7dc56a3 Author: Hao Chen <[email protected]> Authored: Mon Feb 22 19:06:23 2016 +0800 Committer: Li, Yang <[email protected]> Committed: Tue Feb 23 11:31:54 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/util/DateFormat.java | 1 + .../kylin/metadata/model/PartitionDesc.java | 66 +++++++++++++---- .../DefaultPartitionConditionBuilderTest.java | 77 ++++++++++++++++++++ 3 files changed, 130 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/6678ae37/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java index e2777e5..b487dca 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap; public class DateFormat { public static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd"; + public static final String DEFAULT_TIME_PATTERN = "HH:mm:ss"; public static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss"; public static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS"; public static final String[] SUPPORTED_DATETIME_PATTERN = { DEFAULT_DATE_PATTERN, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS, "yyyyMMdd" }; http://git-wip-us.apache.org/repos/asf/kylin/blob/6678ae37/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java index 6194df3..2556743 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java @@ -43,11 +43,18 @@ public class PartitionDesc { @JsonProperty("partition_date_column") private String partitionDateColumn; - + + @JsonProperty("partition_time_column") + private String partitionTimeColumn; + @JsonProperty("partition_date_start") private long partitionDateStart = 0L; @JsonProperty("partition_date_format") private String partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN; + + @JsonProperty("partition_time_format") + private String partitionTimeFormat = DateFormat.DEFAULT_TIME_PATTERN; + @JsonProperty("partition_type") private PartitionType partitionType = PartitionType.APPEND; @JsonProperty("partition_condition_builder") @@ -93,6 +100,14 @@ public class PartitionDesc { this.partitionDateColumn = partitionDateColumn; } + public String getPartitionTimeColumn() { + return partitionTimeColumn; + } + + public void setPartitionTimeColumn(String partitionTimeColumn) { + this.partitionTimeColumn = partitionTimeColumn; + } + @Deprecated public long getPartitionDateStart() { return partitionDateStart; @@ -111,6 +126,14 @@ public class PartitionDesc { this.partitionDateFormat = partitionDateFormat; } + public String getPartitionTimeFormat() { + return partitionTimeFormat; + } + + public void setPartitionTimeFormat(String partitionTimeFormat) { + this.partitionTimeFormat = partitionTimeFormat; + } + public PartitionType getCubePartitionType() { return partitionType; } @@ -137,27 +160,42 @@ public class PartitionDesc { @Override public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive, Map<String, String> tableAlias) { - String partitionColumnName = partDesc.getPartitionDateColumn(); + StringBuilder builder = new StringBuilder(); + String partitionDateColumnName = partDesc.getPartitionDateColumn(); + String partitionTimeColumnName = partDesc.getPartitionTimeColumn(); - // convert to use table alias - int indexOfDot = partitionColumnName.lastIndexOf("."); + if (partitionDateColumnName != null) { + buildSingleColumnRangeCondition(builder, partitionDateColumnName, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), tableAlias); + } + if (partitionTimeColumnName != null) { + if (partitionDateColumnName != null) + builder.append(" AND "); + buildSingleColumnRangeCondition(builder, partitionTimeColumnName, startInclusive, endExclusive, partDesc.getPartitionTimeFormat(), tableAlias); + } + return builder.toString(); + } + + /** + * Convert to use table alias + */ + private static String replaceColumnNameWithAlias(String columnName, Map<String, String> tableAlias) { + int indexOfDot = columnName.lastIndexOf("."); if (indexOfDot > 0) { - String partitionTableName = partitionColumnName.substring(0, indexOfDot); + String partitionTableName = columnName.substring(0, indexOfDot); if (tableAlias != null && tableAlias.containsKey(partitionTableName)) - partitionColumnName = tableAlias.get(partitionTableName) + partitionColumnName.substring(indexOfDot); + columnName = tableAlias.get(partitionTableName) + columnName.substring(indexOfDot); } + return columnName; + } - StringBuilder builder = new StringBuilder(); - + private static void buildSingleColumnRangeCondition(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, String partitionColumnDateFormat, Map<String, String> tableAlias) { + partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias); if (startInclusive > 0) { - builder.append(partitionColumnName + " >= '" + DateFormat.formatToDateStr(startInclusive, partDesc.getPartitionDateFormat()) + "' "); - builder.append("AND "); + builder.append(partitionColumnName + " >= '" + DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat) + "'"); + builder.append(" AND "); } - builder.append(partitionColumnName + " < '" + DateFormat.formatToDateStr(endExclusive, partDesc.getPartitionDateFormat()) + "'"); - - return builder.toString(); + builder.append(partitionColumnName + " < '" + DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat) + "'"); } - } /** http://git-wip-us.apache.org/repos/asf/kylin/blob/6678ae37/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java new file mode 100644 index 0000000..1829250 --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.metadata.model; + +import java.util.HashMap; + +import org.apache.kylin.common.util.DateFormat; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class DefaultPartitionConditionBuilderTest { + private PartitionDesc.DefaultPartitionConditionBuilder partitionConditionBuilder; + + @Before + public void setUp() { + partitionConditionBuilder = new PartitionDesc.DefaultPartitionConditionBuilder(); + } + + @Test + public void testDatePartition() { + PartitionDesc partitionDesc = new PartitionDesc(); + partitionDesc.setPartitionDateColumn("DEFAULT.TABLE_NAME.DATE_COLUMN"); + partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); + String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23"), new HashMap<String, String>()); + Assert.assertEquals("DEFAULT.TABLE_NAME.DATE_COLUMN >= '2016-02-22' AND DEFAULT.TABLE_NAME.DATE_COLUMN < '2016-02-23'", condition); + } + + @Test + public void testTimePartition() { + PartitionDesc partitionDesc = new PartitionDesc(); + partitionDesc.setPartitionTimeColumn("DEFAULT.TABLE_NAME.HOUR_COLUMN"); + partitionDesc.setPartitionTimeFormat("HH"); + String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS), DateFormat.stringToMillis("2016-02-23 01:00:00", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS), new HashMap<String, String>()); + Assert.assertEquals("DEFAULT.TABLE_NAME.HOUR_COLUMN >= '00' AND DEFAULT.TABLE_NAME.HOUR_COLUMN < '01'", condition); + } + + @Test + public void testDateAndTimePartition() { + PartitionDesc partitionDesc = new PartitionDesc(); + partitionDesc.setPartitionDateColumn("DEFAULT.TABLE_NAME.DATE_COLUMN"); + partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); + partitionDesc.setPartitionTimeColumn("DEFAULT.TABLE_NAME.HOUR_COLUMN"); + partitionDesc.setPartitionTimeFormat("H"); + String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS), DateFormat.stringToMillis("2016-02-23 01:00:00", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS), new HashMap<String, String>()); + Assert.assertEquals("DEFAULT.TABLE_NAME.DATE_COLUMN >= '2016-02-22' AND DEFAULT.TABLE_NAME.DATE_COLUMN < '2016-02-23' AND DEFAULT.TABLE_NAME.HOUR_COLUMN >= '0' AND DEFAULT.TABLE_NAME.HOUR_COLUMN < '1'", condition); + } + + @Test + public void testDateAndTimePartitionWithAlias() { + PartitionDesc partitionDesc = new PartitionDesc(); + partitionDesc.setPartitionDateColumn("TABLE_ALIAS.DATE_COLUMN"); + partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); + partitionDesc.setPartitionTimeColumn("TABLE_ALIAS.HOUR_COLUMN"); + partitionDesc.setPartitionTimeFormat("H"); + String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS), DateFormat.stringToMillis("2016-02-23 01:00:00", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS), new HashMap<String, String>() { + { + put("TABLE_ALIAS", "DEFAULT.TABLE_NAME"); + } + }); + Assert.assertEquals("DEFAULT.TABLE_NAME.DATE_COLUMN >= '2016-02-22' AND DEFAULT.TABLE_NAME.DATE_COLUMN < '2016-02-23' AND DEFAULT.TABLE_NAME.HOUR_COLUMN >= '0' AND DEFAULT.TABLE_NAME.HOUR_COLUMN < '1'", condition); + } +} \ No newline at end of file
