[
https://issues.apache.org/jira/browse/KYLIN-5371?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17652159#comment-17652159
]
ASF GitHub Bot commented on KYLIN-5371:
---------------------------------------
liuzhao-lz commented on code in PR #2049:
URL: https://github.com/apache/kylin/pull/2049#discussion_r1057488331
##########
core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java:
##########
@@ -194,4 +202,18 @@ public static boolean isDatePattern(String ptn) {
return COMPACT_DATE_PATTERN.equals(ptn) || YYYYMMDDHH.equals(ptn) ||
YYYYMMDDHHMM.equals(ptn)
|| YYYYMMDDHHMMSS.equals(ptn);
}
+
+ public static Long getFormatTimeStamp(long time, String pattern) {
+ try {
+ if (StringUtils.isNotBlank(pattern)) {
+ SimpleDateFormat sdf = new SimpleDateFormat(pattern,
Locale.getDefault(Locale.Category.FORMAT));
+ sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
Review Comment:
##########
kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala:
##########
@@ -366,7 +366,9 @@ class FilePruner(cubeInstance: CubeInstance,
val pruned = segDirs.filter {
e => {
val tsRange = cubeInstance.getSegment(e.segmentName,
SegmentStatusEnum.READY).getTSRange
- SegFilters(tsRange.startValue, tsRange.endValue, pattern)
+ // tsRange:
20221219000000_20221219010000、20221219010000_20221219020000, pattern: yyyy-MM-dd
+ val start = DateFormat.getFormatTimeStamp(tsRange.startValue,
pattern)
+ SegFilters(start, tsRange.endValue, pattern)
Review Comment:
tsRange.endValue 不需要考虑，DateFormat.getFormatTimeStamp(tsRange.startValue,
pattern) 这个的作用是将 startValue 格式化成天的时间戳值（忽略小时），原因是 SegFilters 的 foldFilter
中是按天格式化where 分区字段值的（记作ts），后续比较也是 “ts >= start && ts < end” end
如果是到时间其对应的时间戳一定是大于天的时间戳值因而可以不用处理。
> Kylin4 在多分区查询bug
> ----------------
>
> Key: KYLIN-5371
> URL: https://issues.apache.org/jira/browse/KYLIN-5371
> Project: Kylin
> Issue Type: Bug
> Affects Versions: v4.0.1, v4.0.2
> Reporter: Liu Zhao
> Assignee: Liu Zhao
> Priority: Major
> Attachments: image-2022-12-19-11-33-36-654.png,
> image-2022-12-19-11-34-06-372.png, image-2022-12-19-11-34-45-932.png,
> image-2022-12-19-11-35-03-652.png, image-2022-12-19-11-49-48-323.png
>
>
> 在创建model时如果增量构建时指定了两个partition列，date 和 hour，构建没有问题，但在查询时如果where只指定 =
> 某个date值，查询结果非预期值。
> // pdate, phour 都是分区列，在创建model时也指定为partition，详情见附件图片
> --q1:
> select pdate, phour, count(1) from lz_test_partition where pdate =
> '2022-12-19' group by pdate, phour
> --q2:
> select pdate, phour, count(1) from lz_test_partition group by pdate, phour
> 查看源码，bug 出现在 org.apache.spark.sql.execution.datasource.SegFilters#foldFilter
> 和 org.apache.spark.sql.execution.datasource.SegFilters#insurance
> 中，一处只用日期判断一处用到time级判断。
> {code:java}
> case class SegFilters(start: Long, end: Long, pattern: String) extends
> Logging {
> private def insurance(value: Any)
> (func: Long => Filter): Filter = {
> value match {
> case v: Date =>
> // see SPARK-27546
> val ts = DateFormat.stringToMillis(v.toString)
> func(ts)
> case v @ (_:String | _: Int | _: Long) if pattern != null =>
> val format = DateFormat.getDateFormat(pattern)
> val time = format.parse(v.toString).getTime
> func(time)
> case v: Timestamp =>
> func(v.getTime)
> case _ =>
> Trivial(true)
> }
> }
> /**
> * Recursively fold provided filters to trivial,
> * blocks are always non-empty.
> */
> def foldFilter(filter: Filter): Filter = {
> filter match {
> case EqualTo(_, value: Any) =>
> insurance(value) {
> ts => Trivial(ts >= start && ts < end) --注意在这个地方是有问题的，ts
> 是date，但start 和 end 可以是到time级，因此在这里的过滤会丢
> }
> case In(_, values: Array[Any]) =>
> val satisfied = values.map(v => insurance(v) {
> ts => Trivial(ts >= start && ts < end)
> }).exists(_.equals(Trivial(true)))
> Trivial(satisfied)
> case IsNull(_) =>
> Trivial(false)
> case IsNotNull(_) =>
> Trivial(true)
> case GreaterThan(_, value: Any) =>
> insurance(value) {
> ts => Trivial(ts < end)
> }
> case GreaterThanOrEqual(_, value: Any) =>
> insurance(value) {
> ts => Trivial(ts < end)
> }
> case LessThan(_, value: Any) =>
> insurance(value) {
> ts => Trivial(ts > start)
> }
> case LessThanOrEqual(_, value: Any) =>
> insurance(value) {
> ts => Trivial(ts >= start)
> }
> case And(left: Filter, right: Filter) =>
> And(foldFilter(left), foldFilter(right)) match {
> case And(AlwaysFalse, _) => Trivial(false)
> case And(_, AlwaysFalse) => Trivial(false)
> case And(AlwaysTrue, right) => right
> case And(left, AlwaysTrue) => left
> case other => other
> }
> case Or(left: Filter, right: Filter) =>
> Or(foldFilter(left), foldFilter(right)) match {
> case Or(AlwaysTrue, _) => Trivial(true)
> case Or(_, AlwaysTrue) => Trivial(true)
> case Or(AlwaysFalse, right) => right
> case Or(left, AlwaysFalse) => left
> case other => other
> }
> case unsupportedFilter =>
> // return 'true' to scan all partitions
> // currently unsupported filters are:
> // - StringStartsWith
> // - StringEndsWith
> // - StringContains
> // - EqualNullSafe
> Trivial(true)
> }
> }
> def Trivial(value: Boolean): Filter = {
> if (value) AlwaysTrue else AlwaysFalse
> }
> }
> {code}
> 详情及原因看附件图片：
> !image-2022-12-19-11-49-48-323.png!
> !image-2022-12-19-11-33-36-654.png!
> !image-2022-12-19-11-34-06-372.png!
> !image-2022-12-19-11-34-45-932.png!
> !image-2022-12-19-11-35-03-652.png!
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
