Wayne1c closed pull request #356: KYLIN-3665 Partition time column may never be
used
URL: https://github.com/apache/kylin/pull/356
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index dcb37ecdad..47802d9831 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -95,6 +95,14 @@ public boolean partitionColumnIsTimeMillis() {
return type.isBigInt() &&
!DateFormat.isDatePattern(partitionDateFormat);
}
+ public boolean partitionTimeColumnIsInt() {
+ if (partitionTimeColumnRef == null)
+ return false;
+
+ DataType type = partitionTimeColumnRef.getType();
+ return (type.isInt() || type.isBigInt());
+ }
+
public boolean isPartitioned() {
return partitionDateColumnRef != null;
}
@@ -194,103 +202,99 @@ public String buildDateRangeCondition(PartitionDesc
partDesc, ISegment seg, Segm
long startInclusive = (Long) segRange.start.v;
long endExclusive = (Long) segRange.end.v;
+ if (endExclusive <= startInclusive) {
+ return "1=0";
+ }
+
TblColRef partitionDateColumn =
partDesc.getPartitionDateColumnRef();
TblColRef partitionTimeColumn =
partDesc.getPartitionTimeColumnRef();
StringBuilder builder = new StringBuilder();
- if (partDesc.partitionColumnIsYmdInt()) {
- buildSingleColumnRangeCondAsYmdInt(builder,
partitionDateColumn, startInclusive, endExclusive,
- partDesc.getPartitionDateFormat());
- } else if (partDesc.partitionColumnIsTimeMillis()) {
- buildSingleColumnRangeCondAsTimeMillis(builder,
partitionDateColumn, startInclusive, endExclusive);
- } else if (partitionDateColumn != null && partitionTimeColumn ==
null) {
- buildSingleColumnRangeCondition(builder, partitionDateColumn,
startInclusive, endExclusive,
- partDesc.getPartitionDateFormat());
- } else if (partitionDateColumn == null && partitionTimeColumn !=
null) {
- buildSingleColumnRangeCondition(builder, partitionTimeColumn,
startInclusive, endExclusive,
- partDesc.getPartitionTimeFormat());
- } else if (partitionDateColumn != null && partitionTimeColumn !=
null) {
- buildMultipleColumnRangeCondition(builder,
partitionDateColumn, partitionTimeColumn, startInclusive,
- endExclusive, partDesc.getPartitionDateFormat(),
partDesc.getPartitionTimeFormat());
+ if (partitionDateColumn != null && partitionTimeColumn != null) {
+ buildMultipleColumnRangeCondition(builder,
partitionDateColumn, partitionTimeColumn, startInclusive, endExclusive,
partDesc);
+ } else if (partitionDateColumn != null) {
+ buildSingleColumnRangeCondition(builder, partitionDateColumn,
startInclusive, endExclusive, partDesc, true);
+ } else if (partitionTimeColumn != null) {
+ buildSingleColumnRangeCondition(builder, partitionTimeColumn,
startInclusive, endExclusive, partDesc, false);
}
return builder.toString();
}
- private static void
buildSingleColumnRangeCondAsTimeMillis(StringBuilder builder, TblColRef
partitionColumn,
- long startInclusive, long endExclusive) {
- String partitionColumnName = partitionColumn.getIdentity();
- builder.append(partitionColumnName + " >= " + startInclusive);
- builder.append(" AND ");
- builder.append(partitionColumnName + " < " + endExclusive);
- }
-
- private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder
builder, TblColRef partitionColumn,
- long startInclusive, long endExclusive, String
partitionColumnDateFormat) {
- String partitionColumnName = partitionColumn.getIdentity();
- builder.append(partitionColumnName + " >= "
- + DateFormat.formatToDateStr(startInclusive,
partitionColumnDateFormat));
- builder.append(" AND ");
- builder.append(
- partitionColumnName + " < " +
DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat));
- }
-
private static void buildSingleColumnRangeCondition(StringBuilder
builder, TblColRef partitionColumn,
- long startInclusive, long endExclusive, String
partitionColumnDateFormat) {
+ long startInclusive, long endExclusive, PartitionDesc
partitionDesc, boolean isPartitionDateColumn) {
String partitionColumnName = partitionColumn.getIdentity();
- if (endExclusive <= startInclusive) {
- builder.append("1=0");
- return;
- }
-
String startInc = null;
String endInc = null;
- if (StringUtils.isBlank(partitionColumnDateFormat)) {
- startInc = String.valueOf(startInclusive);
- endInc = String.valueOf(endExclusive);
+
+ if (isPartitionDateColumn) {
+ startInc = convertDateConditionValue(startInclusive,
partitionDesc);
+ endInc = convertDateConditionValue(endExclusive,
partitionDesc);
} else {
- startInc = DateFormat.formatToDateStr(startInclusive,
partitionColumnDateFormat);
- endInc = DateFormat.formatToDateStr(endExclusive,
partitionColumnDateFormat);
+ startInc = convertTimeConditionValue(startInclusive,
partitionDesc);
+ endInc = convertTimeConditionValue(endExclusive,
partitionDesc);
}
- builder.append(partitionColumnName + " >= '" + startInc + "'");
+ builder.append(partitionColumnName + " >= " + startInc);
builder.append(" AND ");
- builder.append(partitionColumnName + " < '" + endInc + "'");
+ builder.append(partitionColumnName + " < " + endInc);
+ }
+
+ private static String convertDateConditionValue(long date,
PartitionDesc partitionDesc) {
+ if(partitionDesc.partitionColumnIsYmdInt()) {
+ return DateFormat.formatToDateStr(date,
partitionDesc.getPartitionDateFormat());
+ } else if (partitionDesc.partitionColumnIsTimeMillis()) {
+ return String.valueOf(date);
+ } else {
+ return "'" + DateFormat.formatToDateStr(date,
partitionDesc.getPartitionDateFormat()) + "'";
+ }
+ }
+
+ private static String convertTimeConditionValue(long time,
PartitionDesc partitionDesc) {
+ //currently supported time format: HH:mm:ss、HH:mm、HH(String/int)
+ //TODO: HHmmss、HHmm(String/int)
+
+ if (partitionDesc.partitionTimeColumnIsInt()) {
+ return DateFormat.formatToDateStr(time,
partitionDesc.getPartitionTimeFormat());
+ } else {
+ return "'" + DateFormat.formatToDateStr(time,
partitionDesc.getPartitionTimeFormat()) + "'";
+ }
}
private static void buildMultipleColumnRangeCondition(StringBuilder
builder, TblColRef partitionDateColumn,
- TblColRef partitionTimeColumn, long startInclusive, long
endExclusive, String partitionColumnDateFormat,
- String partitionColumnTimeFormat) {
+ TblColRef partitionTimeColumn, long startInclusive, long
endExclusive, PartitionDesc partitionDesc) {
String partitionDateColumnName = partitionDateColumn.getIdentity();
String partitionTimeColumnName = partitionTimeColumn.getIdentity();
+
+ String conditionDateStartValue =
convertDateConditionValue(startInclusive, partitionDesc);
+ String conditionDateEndValue =
convertDateConditionValue(endExclusive, partitionDesc);
+
+ String conditionTimeStartValue =
convertTimeConditionValue(startInclusive, partitionDesc);
+ String conditionTimeEndValue =
convertTimeConditionValue(endExclusive, partitionDesc);
+
builder.append("(");
builder.append("(");
- builder.append(partitionDateColumnName + " = '"
- + DateFormat.formatToDateStr(startInclusive,
partitionColumnDateFormat) + "'").append(" AND ")
- .append(partitionTimeColumnName + " >= '"
- + DateFormat.formatToDateStr(startInclusive,
partitionColumnTimeFormat) + "'");
+ builder.append(partitionDateColumnName + " = " +
conditionDateStartValue).append(" AND ")
+ .append(partitionTimeColumnName + " >= " +
conditionTimeStartValue);
builder.append(")");
builder.append(" OR ");
builder.append("(");
- builder.append(partitionDateColumnName + " > '"
- + DateFormat.formatToDateStr(startInclusive,
partitionColumnDateFormat) + "'");
+ builder.append(partitionDateColumnName + " > " +
conditionDateStartValue);
builder.append(")");
builder.append(")");
+
builder.append(" AND ");
builder.append("(");
builder.append("(");
- builder.append(partitionDateColumnName + " = '"
- + DateFormat.formatToDateStr(endExclusive,
partitionColumnDateFormat) + "'").append(" AND ")
- .append(partitionTimeColumnName + " < '"
- + DateFormat.formatToDateStr(endExclusive,
partitionColumnTimeFormat) + "'");
+ builder.append(partitionDateColumnName + " = " +
conditionDateEndValue).append(" AND ")
+ .append(partitionTimeColumnName + " < " +
conditionTimeEndValue);
builder.append(")");
builder.append(" OR ");
builder.append("(");
- builder.append(partitionDateColumnName + " < '"
- + DateFormat.formatToDateStr(endExclusive,
partitionColumnDateFormat) + "'");
+ builder.append(partitionDateColumnName + " < " +
conditionDateEndValue);
builder.append(")");
builder.append(")");
}
diff --git
a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
index b536e29191..403c2f85c1 100644
---
a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
+++
b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
@@ -51,47 +51,139 @@ public void testDatePartition() {
TblColRef col =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN",
"string");
partitionDesc.setPartitionDateColumnRef(col);
partitionDesc.setPartitionDateColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"),
DateFormat.stringToMillis("2016-02-23 21:32:29"));
+ String condition;
+
partitionDesc.setPartitionDateFormat("yyyy-MM-dd");
- TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"),
DateFormat.stringToMillis("2016-02-23"));
- String condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
- Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND
UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'",
- condition);
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND
UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyy-MM-dd HH:mm");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22 00:00'
AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23 21:32'", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMddHH");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016022200' AND
UNKNOWN_ALIAS.DATE_COLUMN < '2016022321'", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMddHHmm");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '201602220000' AND
UNKNOWN_ALIAS.DATE_COLUMN < '201602232132'", condition);
range = new TSRange(0L, 0L);
condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals("1=0", condition);
}
+ @Test
+ public void testDatePartitionYmdInt() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN",
"integer");
+ partitionDesc.setPartitionDateColumnRef(col);
+ partitionDesc.setPartitionDateColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"),
DateFormat.stringToMillis("2016-02-23 20:32:29"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyyMMdd");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= 20160222 AND
UNKNOWN_ALIAS.DATE_COLUMN < 20160223", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMddHHmmss");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= 20160222000000 AND
UNKNOWN_ALIAS.DATE_COLUMN < 20160223203229", condition);
+ }
+
+ @Test
+ public void testDatePartitionTimeMillis() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN",
"bigint");
+ partitionDesc.setPartitionDateColumnRef(col);
+ partitionDesc.setPartitionDateColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"),
DateFormat.stringToMillis("2016-02-23 20:32:29"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyy-MM-dd HH:mm:ss");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= 1456099200000 AND
UNKNOWN_ALIAS.DATE_COLUMN < 1456259549000", condition);
+ }
+
@Test
public void testTimePartition() {
PartitionDesc partitionDesc = new PartitionDesc();
TblColRef col =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN",
"string");
partitionDesc.setPartitionTimeColumnRef(col);
partitionDesc.setPartitionTimeColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
00:12:00"), DateFormat.stringToMillis("2016-02-23 01:00:00"));
+ String condition;
+
partitionDesc.setPartitionTimeFormat("HH");
- TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
00:00:00"),
- DateFormat.stringToMillis("2016-02-23 01:00:00"));
- String condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND
UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition);
+
+ partitionDesc.setPartitionTimeFormat("HH:mm");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00:12' AND
UNKNOWN_ALIAS.HOUR_COLUMN < '01:00'", condition);
}
@Test
- public void testDateAndTimePartition() {
+ public void testTimePartitionIsInt() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN",
"integer");
+ partitionDesc.setPartitionTimeColumnRef(col);
+ partitionDesc.setPartitionTimeColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
19:12:00"), DateFormat.stringToMillis("2016-02-23 20:00:00"));
+ String condition;
+
+ partitionDesc.setPartitionTimeFormat("HH");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= 19 AND
UNKNOWN_ALIAS.HOUR_COLUMN < 20", condition);
+ }
+
+ @Test
+ public void testDateAndTimePartitionString() {
PartitionDesc partitionDesc = new PartitionDesc();
TblColRef col1 =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN",
"string");
partitionDesc.setPartitionDateColumnRef(col1);
partitionDesc.setPartitionDateColumn(col1.getCanonicalName());
- partitionDesc.setPartitionDateFormat("yyyy-MM-dd");
TblColRef col2 =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN",
"string");
partitionDesc.setPartitionTimeColumnRef(col2);
partitionDesc.setPartitionTimeColumn(col2.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
00:00:00"), DateFormat.stringToMillis("2016-02-23 12:23:00"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyy-MM-dd");
partitionDesc.setPartitionTimeFormat("H");
- TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
00:00:00"),
- DateFormat.stringToMillis("2016-02-23 01:00:00"));
- String condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals(
+ "((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND
UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN >
'2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND
UNKNOWN_ALIAS.HOUR_COLUMN < '12') OR (UNKNOWN_ALIAS.DATE_COLUMN <
'2016-02-23'))",
+ condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMdd");
+ partitionDesc.setPartitionTimeFormat("HH:mm");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals(
- "((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND
UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN >
'2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND
UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN <
'2016-02-23'))",
+ "((UNKNOWN_ALIAS.DATE_COLUMN = '20160222' AND
UNKNOWN_ALIAS.HOUR_COLUMN >= '00:00') OR (UNKNOWN_ALIAS.DATE_COLUMN >
'20160222')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '20160223' AND
UNKNOWN_ALIAS.HOUR_COLUMN < '12:23') OR (UNKNOWN_ALIAS.DATE_COLUMN <
'20160223'))",
condition);
}
+ @Test
+ public void testDateAndTimePartitionInt() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col1 =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN",
"integer");
+ partitionDesc.setPartitionDateColumnRef(col1);
+ partitionDesc.setPartitionDateColumn(col1.getCanonicalName());
+ TblColRef col2 =
TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN",
"bigint");
+ partitionDesc.setPartitionTimeColumnRef(col2);
+ partitionDesc.setPartitionTimeColumn(col2.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22
00:00:00"), DateFormat.stringToMillis("2016-02-23 12:23:00"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyyMMdd");
+ partitionDesc.setPartitionTimeFormat("HH");
+ condition =
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals(
+ "((UNKNOWN_ALIAS.DATE_COLUMN = 20160222 AND
UNKNOWN_ALIAS.HOUR_COLUMN >= 00) OR (UNKNOWN_ALIAS.DATE_COLUMN > 20160222)) AND
((UNKNOWN_ALIAS.DATE_COLUMN = 20160223 AND UNKNOWN_ALIAS.HOUR_COLUMN < 12) OR
(UNKNOWN_ALIAS.DATE_COLUMN < 20160223))",
+ condition);
+
+ }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
