[
https://issues.apache.org/jira/browse/KYLIN-5371?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Liu Zhao reassigned KYLIN-5371:
-------------------------------
Assignee: Liu Zhao
> Kylin4 在多分区查询bug
> ----------------
>
> Key: KYLIN-5371
> URL: https://issues.apache.org/jira/browse/KYLIN-5371
> Project: Kylin
> Issue Type: Bug
> Affects Versions: v4.0.1, v4.0.2
> Reporter: Liu Zhao
> Assignee: Liu Zhao
> Priority: Major
> Attachments: image-2022-12-19-11-33-36-654.png,
> image-2022-12-19-11-34-06-372.png, image-2022-12-19-11-34-45-932.png,
> image-2022-12-19-11-35-03-652.png, image-2022-12-19-11-49-48-323.png
>
>
> 在创建model时如果增量构建时指定了两个partition列，date 和 hour，构建没有问题，但在查询时如果where只指定 =
> 某个date值，查询结果非预期值。
> // pdate, phour 都是分区列，在创建model时也指定为partition，详情见附件图片
> --q1:
> select pdate, phour, count(1) from lz_test_partition where pdate =
> '2022-12-19' group by pdate, phour
> --q2:
> select pdate, phour, count(1) from lz_test_partition group by pdate, phour
> 查看源码，bug 出现在 org.apache.spark.sql.execution.datasource.SegFilters#foldFilter
> 和 org.apache.spark.sql.execution.datasource.SegFilters#insurance
> 中，一处只用日期判断一处用到time级判断。
> {code:java}
> case class SegFilters(start: Long, end: Long, pattern: String) extends
> Logging {
> private def insurance(value: Any)
> (func: Long => Filter): Filter = {
> value match {
> case v: Date =>
> // see SPARK-27546
> val ts = DateFormat.stringToMillis(v.toString)
> func(ts)
> case v @ (_:String | _: Int | _: Long) if pattern != null =>
> val format = DateFormat.getDateFormat(pattern)
> val time = format.parse(v.toString).getTime
> func(time)
> case v: Timestamp =>
> func(v.getTime)
> case _ =>
> Trivial(true)
> }
> }
> /**
> * Recursively fold provided filters to trivial,
> * blocks are always non-empty.
> */
> def foldFilter(filter: Filter): Filter = {
> filter match {
> case EqualTo(_, value: Any) =>
> insurance(value) {
> ts => Trivial(ts >= start && ts < end) --注意在这个地方是有问题的，ts
> 是date，但start 和 end 可以是到time级，因此在这里的过滤会丢
> }
> case In(_, values: Array[Any]) =>
> val satisfied = values.map(v => insurance(v) {
> ts => Trivial(ts >= start && ts < end)
> }).exists(_.equals(Trivial(true)))
> Trivial(satisfied)
> case IsNull(_) =>
> Trivial(false)
> case IsNotNull(_) =>
> Trivial(true)
> case GreaterThan(_, value: Any) =>
> insurance(value) {
> ts => Trivial(ts < end)
> }
> case GreaterThanOrEqual(_, value: Any) =>
> insurance(value) {
> ts => Trivial(ts < end)
> }
> case LessThan(_, value: Any) =>
> insurance(value) {
> ts => Trivial(ts > start)
> }
> case LessThanOrEqual(_, value: Any) =>
> insurance(value) {
> ts => Trivial(ts >= start)
> }
> case And(left: Filter, right: Filter) =>
> And(foldFilter(left), foldFilter(right)) match {
> case And(AlwaysFalse, _) => Trivial(false)
> case And(_, AlwaysFalse) => Trivial(false)
> case And(AlwaysTrue, right) => right
> case And(left, AlwaysTrue) => left
> case other => other
> }
> case Or(left: Filter, right: Filter) =>
> Or(foldFilter(left), foldFilter(right)) match {
> case Or(AlwaysTrue, _) => Trivial(true)
> case Or(_, AlwaysTrue) => Trivial(true)
> case Or(AlwaysFalse, right) => right
> case Or(left, AlwaysFalse) => left
> case other => other
> }
> case unsupportedFilter =>
> // return 'true' to scan all partitions
> // currently unsupported filters are:
> // - StringStartsWith
> // - StringEndsWith
> // - StringContains
> // - EqualNullSafe
> Trivial(true)
> }
> }
> def Trivial(value: Boolean): Filter = {
> if (value) AlwaysTrue else AlwaysFalse
> }
> }
> {code}
> 详情及原因看附件图片：
> !image-2022-12-19-11-49-48-323.png!
> !image-2022-12-19-11-33-36-654.png!
> !image-2022-12-19-11-34-06-372.png!
> !image-2022-12-19-11-34-45-932.png!
> !image-2022-12-19-11-35-03-652.png!
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
