[
https://issues.apache.org/jira/browse/KYLIN-3665?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16699802#comment-16699802
]
ASF GitHub Bot commented on KYLIN-3665:
---------------------------------------
Wayne1c closed pull request #356: KYLIN-3665 Partition time column may never be
used
URL: https://github.com/apache/kylin/pull/356
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index dcb37ecdad..47802d9831 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -95,6 +95,14 @@ public boolean partitionColumnIsTimeMillis() {
return type.isBigInt() &&
!DateFormat.isDatePattern(partitionDateFormat);
}
+ public boolean partitionTimeColumnIsInt() {
+ if (partitionTimeColumnRef == null)
+ return false;
+
+ DataType type = partitionTimeColumnRef.getType();
+ return (type.isInt() || type.isBigInt());
+ }
+
public boolean isPartitioned() {
return partitionDateColumnRef != null;
}
@@ -194,103 +202,99 @@ public String buildDateRangeCondition(PartitionDesc
partDesc, ISegment seg, Segm
long startInclusive = (Long) segRange.start.v;
long endExclusive = (Long) segRange.end.v;
+ if (endExclusive <= startInclusive) {
+ return "1=0";
+ }
+
TblColRef partitionDateColumn =
partDesc.getPartitionDateColumnRef();
TblColRef partitionTimeColumn =
partDesc.getPartitionTimeColumnRef();
StringBuilder builder = new StringBuilder();
- if (partDesc.partitionColumnIsYmdInt()) {
- buildSingleColumnRangeCondAsYmdInt(builder,
partitionDateColumn, startInclusive, endExclusive,
- partDesc.getPartitionDateFormat());
- } else if (partDesc.partitionColumnIsTimeMillis()) {
- buildSingleColumnRangeCondAsTimeMillis(builder,
partitionDateColumn, startInclusive, endExclusive);
- } else if (partitionDateColumn != null && partitionTimeColumn ==
null) {
- buildSingleColumnRangeCondition(builder, partitionDateColumn,
startInclusive, endExclusive,
- partDesc.getPartitionDateFormat());
- } else if (partitionDateColumn == null && partitionTimeColumn !=
null) {
- buildSingleColumnRangeCondition(builder, partitionTimeColumn,
startInclusive, endExclusive,
- partDesc.getPartitionTimeFormat());
- } else if (partitionDateColumn != null && partitionTimeColumn !=
null) {
- buildMultipleColumnRangeCondition(builder,
partitionDateColumn, partitionTimeColumn, startInclusive,
- endExclusive, partDesc.getPartitionDateFormat(),
partDesc.getPartitionTimeFormat());
+ if (partitionDateColumn != null && partitionTimeColumn != null) {
+ buildMultipleColumnRangeCondition(builder,
partitionDateColumn, partitionTimeColumn, startInclusive, endExclusive,
partDesc);
+ } else if (partitionDateColumn != null) {
+ buildSingleColumnRangeCondition(builder, partitionDateColumn,
startInclusive, endExclusive, partDesc, true);
+ } else if (partitionTimeColumn != null) {
+ buildSingleColumnRangeCondition(builder, partitionTimeColumn,
startInclusive, endExclusive, partDesc, false);
}
return builder.toString();
}
- private static void
buildSingleColumnRangeCondAsTimeMillis(StringBuilder builder, TblColRef
partitionColumn,
- long startInclusive, long endExclusive) {
- String partitionColumnName = partitionColumn.getIdentity();
- builder.append(partitionColumnName + " >= " + startInclusive);
- builder.append(" AND ");
- builder.append(partitionColumnName + " < " + endExclusive);
- }
-
- private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder
builder, TblColRef partitionColumn,
- long startInclusive, long endExclusive, String
partitionColumnDateFormat) {
- String partitionColumnName = partitionColumn.getIdentity();
- builder.append(partitionColumnName + " >= "
- + DateFormat.formatToDateStr(startInclusive,
partitionColumnDateFormat));
- builder.append(" AND ");
- builder.append(
- partitionColumnName + " < " +
DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat));
- }
-
private static void buildSingleColumnRangeCondition(StringBuilder
builder, TblColRef partitionColumn,
- long startInclusive, long endExclusive, String
partitionColumnDateFormat) {
+ long startInclusive, long endExclusive, PartitionDesc
partitionDesc, boolean isPartitionDateColumn) {
String partitionColumnName = partitionColumn.getIdentity();
- if (endExclusive <= startInclusive) {
- builder.append("1=0");
- return;
- }
-
String startInc = null;
String endInc = null;
- if (StringUtils.isBlank(partitionColumnDateFormat)) {
- startInc = String.valueOf(startInclusive);
- endInc = String.valueOf(endExclusive);
+
+ if (isPartitionDateColumn) {
+ startInc = convertDateConditionValue(startInclusive,
partitionDesc);
+ endInc = convertDateConditionValue(endExclusive,
partitionDesc);
} else {
- startInc = DateFormat.formatToDateStr(startInclusive,
partitionColumnDateFormat);
- endInc = DateFormat.formatToDateStr(endExclusive,
partitionColumnDateFormat);
+ startInc = convertTimeConditionValue(startInclusive,
partitionDesc);
+ endInc = convertTimeConditionValue(endExclusive,
partitionDesc);
}
- builder.append(partitionColumnName + " >= '" + startInc + "'");
+ builder.append(partitionColumnName + " >= " + startInc);
builder.append(" AND ");
- builder.append(partitionColumnName + " < '" + endInc + "'");
+ builder.append(partitionColumnName + " < " + endInc);
+ }
+
+ private static String convertDateConditionValue(long date,
PartitionDesc partitionDesc) {
+ if(partitionDesc.partitionColumnIsYmdInt()) {
+ return DateFormat.formatToDateStr(date,
partitionDesc.getPartitionDateFormat());
+ } else if (partitionDesc.partitionColumnIsTimeMillis()) {
+ return String.valueOf(date);
+ } else {
+ return "'" + DateFormat.formatToDateStr(date,
partitionDesc.getPartitionDateFormat()) + "'";
+ }
+ }
+
+ private static String convertTimeConditionValue(long time,
PartitionDesc partitionDesc) {
+ //currently supported time format: HH:mm:ss、HH:mm、HH(String/int)
+ //TODO: HHmmss、HHmm(String/int)
+
+ if (partitionDesc.partitionTimeColumnIsInt()) {
+ return DateFormat.formatToDateStr(time,
partitionDesc.getPartitionTimeFormat());
+ } else {
+ return "'" + DateFormat.formatToDateStr(time,
partitionDesc.getPartitionTimeFormat()) + "'";
+ }
}
private static void buildMultipleColumnRangeCondition(StringBuilder
builder, TblColRef partitionDateColumn,
- TblColRef partitionTimeColumn, long startInclusive, long
endExclusive, String partitionColumnDateFormat,
- String partitionColumnTimeFormat) {
+ TblColRef partitionTimeColumn, long startInclusive, long
endExclusive, PartitionDesc partitionDesc) {
String partitionDateColumnName = partitionDateColumn.getIdentity();
String partitionTimeColumnName = partitionTimeColumn.getIdentity();
+
+ String conditionDateStartValue =
convertDateConditionValue(startInclusive, partitionDesc);
+ String conditionDateEndValue =
convertDateConditionValue(endExclusive, partitionDesc);
+
+ String conditionTimeStartValue =
convertTimeConditionValue(startInclusive, partitionDesc);
+ String conditionTimeEndValue =
convertTimeConditionValue(endExclusive, partitionDesc);
+
builder.append("(");
builder.append("(");
- builder.append(partitionDateColumnName + " = '"
- + DateFormat.formatToDateStr(startInclusive,
partitionColumnDateFormat) + "'").append(" AND ")
- .append(partitionTimeColumnName + " >= '"
- + DateFormat.formatToDateStr(startInclusive,
partitionColumnTimeFormat) + "'");
+ builder.append(partitionDateColumnName + " = " +
conditionDateStartValue).append(" AND ")
+ .append(partitionTimeColumnName + " >= " +
conditionTimeStartValue);
builder.append(")");
builder.append(" OR ");
builder.append("(");
- builder.append(partitionDateColumnName + " > '"
- + DateFormat.formatToDateStr(startInclusive,
partitionColumnDateFormat) + "'");
+ builder.append(partitionDateColumnName + " > " +
conditionDateStartValue);
builder.append(")");
builder.append(")");
+
builder.append(" AND ");
builder.append("(");
builder.append("(");
- builder.append(partitionDateColumnName + " = '"
- + DateFormat.formatToDateStr(endExclusive,
partitionColumnDateFormat) + "'").append(" AND ")
- .append(partitionTimeColumnName + " < '"
- + DateFormat.formatToDateStr(endExclusive,
partitionColumnTimeFormat) + "'");
+ builder.append(partitionDateColumnName + " = " +
conditionDateEndValue).append(" AND ")
+ .append(partitionTimeColumnName + " < " +
conditionTimeEndValue);
builder.append(")");
builder.append(" OR ");
builder.append("(");
- builder.append(partitionDateColumnName + " < '"
- + DateFormat.formatToDateStr(endExclusive,
partitionColumnDateFormat) + "'");
+ builder.append(partitionDateColumnName + " < " +
conditionDateEndValue);
builder.append(")");
builder.append(")");
}
diff --git
a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
index b536e29191..403c2f85c1 100644
---
a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
+++
b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
@@ -51,47 +51,139 @@ public void testDatePartition() {
TblColRef col =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN",
"string");
partitionDesc.setPartitionDateColumnRef(col);
partitionDesc.setPartitionDateColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"),
DateFormat.stringToMillis("2016-02-23 21:32:29"));
+ String condition;
+
partitionDesc.setPartitionDateFormat("yyyy-MM-dd");
- TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"),
DateFormat.stringToMillis("2016-02-23"));
- String condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
- Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND
UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'",
- condition);
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND
UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyy-MM-dd HH:mm");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22 00:00'
AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23 21:32'", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMddHH");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016022200' AND
UNKNOWN_ALIAS.DATE_COLUMN < '2016022321'", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMddHHmm");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '201602220000' AND
UNKNOWN_ALIAS.DATE_COLUMN < '201602232132'", condition);
range = new TSRange(0L, 0L);
condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals("1=0", condition);
}
+ @Test
+ public void testDatePartitionYmdInt() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN",
"integer");
+ partitionDesc.setPartitionDateColumnRef(col);
+ partitionDesc.setPartitionDateColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"),
DateFormat.stringToMillis("2016-02-23 20:32:29"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyyMMdd");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= 20160222 AND
UNKNOWN_ALIAS.DATE_COLUMN < 20160223", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMddHHmmss");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= 20160222000000 AND
UNKNOWN_ALIAS.DATE_COLUMN < 20160223203229", condition);
+ }
+
+ @Test
+ public void testDatePartitionTimeMillis() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN",
"bigint");
+ partitionDesc.setPartitionDateColumnRef(col);
+ partitionDesc.setPartitionDateColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"),
DateFormat.stringToMillis("2016-02-23 20:32:29"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyy-MM-dd HH:mm:ss");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= 1456099200000 AND
UNKNOWN_ALIAS.DATE_COLUMN < 1456259549000", condition);
+ }
+
@Test
public void testTimePartition() {
PartitionDesc partitionDesc = new PartitionDesc();
TblColRef col =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN",
"string");
partitionDesc.setPartitionTimeColumnRef(col);
partitionDesc.setPartitionTimeColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
00:12:00"), DateFormat.stringToMillis("2016-02-23 01:00:00"));
+ String condition;
+
partitionDesc.setPartitionTimeFormat("HH");
- TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
00:00:00"),
- DateFormat.stringToMillis("2016-02-23 01:00:00"));
- String condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND
UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition);
+
+ partitionDesc.setPartitionTimeFormat("HH:mm");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00:12' AND
UNKNOWN_ALIAS.HOUR_COLUMN < '01:00'", condition);
}
@Test
- public void testDateAndTimePartition() {
+ public void testTimePartitionIsInt() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN",
"integer");
+ partitionDesc.setPartitionTimeColumnRef(col);
+ partitionDesc.setPartitionTimeColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
19:12:00"), DateFormat.stringToMillis("2016-02-23 20:00:00"));
+ String condition;
+
+ partitionDesc.setPartitionTimeFormat("HH");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= 19 AND
UNKNOWN_ALIAS.HOUR_COLUMN < 20", condition);
+ }
+
+ @Test
+ public void testDateAndTimePartitionString() {
PartitionDesc partitionDesc = new PartitionDesc();
TblColRef col1 =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN",
"string");
partitionDesc.setPartitionDateColumnRef(col1);
partitionDesc.setPartitionDateColumn(col1.getCanonicalName());
- partitionDesc.setPartitionDateFormat("yyyy-MM-dd");
TblColRef col2 =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN",
"string");
partitionDesc.setPartitionTimeColumnRef(col2);
partitionDesc.setPartitionTimeColumn(col2.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
00:00:00"), DateFormat.stringToMillis("2016-02-23 12:23:00"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyy-MM-dd");
partitionDesc.setPartitionTimeFormat("H");
- TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
00:00:00"),
- DateFormat.stringToMillis("2016-02-23 01:00:00"));
- String condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals(
+ "((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND
UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN >
'2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND
UNKNOWN_ALIAS.HOUR_COLUMN < '12') OR (UNKNOWN_ALIAS.DATE_COLUMN <
'2016-02-23'))",
+ condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMdd");
+ partitionDesc.setPartitionTimeFormat("HH:mm");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals(
- "((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND
UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN >
'2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND
UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN <
'2016-02-23'))",
+ "((UNKNOWN_ALIAS.DATE_COLUMN = '20160222' AND
UNKNOWN_ALIAS.HOUR_COLUMN >= '00:00') OR (UNKNOWN_ALIAS.DATE_COLUMN >
'20160222')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '20160223' AND
UNKNOWN_ALIAS.HOUR_COLUMN < '12:23') OR (UNKNOWN_ALIAS.DATE_COLUMN <
'20160223'))",
condition);
}
+ @Test
+ public void testDateAndTimePartitionInt() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col1 =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN",
"integer");
+ partitionDesc.setPartitionDateColumnRef(col1);
+ partitionDesc.setPartitionDateColumn(col1.getCanonicalName());
+ TblColRef col2 =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN",
"bigint");
+ partitionDesc.setPartitionTimeColumnRef(col2);
+ partitionDesc.setPartitionTimeColumn(col2.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
00:00:00"), DateFormat.stringToMillis("2016-02-23 12:23:00"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyyMMdd");
+ partitionDesc.setPartitionTimeFormat("HH");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals(
+ "((UNKNOWN_ALIAS.DATE_COLUMN = 20160222 AND
UNKNOWN_ALIAS.HOUR_COLUMN >= 00) OR (UNKNOWN_ALIAS.DATE_COLUMN > 20160222)) AND
((UNKNOWN_ALIAS.DATE_COLUMN = 20160223 AND UNKNOWN_ALIAS.HOUR_COLUMN < 12) OR
(UNKNOWN_ALIAS.DATE_COLUMN < 20160223))",
+ condition);
+
+ }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Partition time column may never be added
> ----------------------------------------
>
> Key: KYLIN-3665
> URL: https://issues.apache.org/jira/browse/KYLIN-3665
> Project: Kylin
> Issue Type: Bug
> Reporter: Chao Long
> Assignee: Chao Long
> Priority: Major
> Fix For: v2.6.0
>
>
> The partition time column will never be added to the WHERE clause if the
> partition date column is type of int or bigint and the date format is
> "yyyyMMdd", "yyyyMMddHH", "yyyyMMddHHmm" or "yyyyMMddHHmmss".
> Let’s say I have a fact table with two date partition column: day as the
> format of integer (yyyyMMdd) and minute as the format of string (HHmm). I
> have models created based on it and set the Partition Date Column to be “day”
> with the format of yyyyMMdd and the Partition Time Column to be “minute” with
> the format of HHmm. When I build the cube built on top of that model, I set
> the Start Date to be 2018-11-01 00:00:00 and the End Date to be 2018-11-01
> 01:00:00. The building process is supposed to retrieve the data from the fact
> table in the range of “day = 20181101 and minute>=’0000’ and minute<’0100’”.
> However, it retrieves the data in the range of “WHERE 1=1 AND (DAY >=
> 20181101 AND DAY < 20181101), so no data are actually retrieved.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
