Sigma-Ma commented on code in PR #174:
URL: https://github.com/apache/hbase-connectors/pull/174#discussion_r4488877647
##########
spark4/hbase-spark4/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseScanBuilder.scala:
##########
@@ -57,33 +57,25 @@ class HBaseScanBuilder(schema: StructType, properties:
Map[String, String])
override def pushFilters(filters: Array[Filter]): Array[Filter] = {
val hasCompositeRowKey = catalog.getRowKey.size > 1
- def isSupported(f: Filter): Boolean = f match {
- case EqualTo(attr, _) => isSupportedField(attr)
- case LessThan(attr, _) => isSupportedField(attr)
- case GreaterThan(attr, _) => isSupportedField(attr)
- case LessThanOrEqual(attr, _) => isSupportedField(attr)
- case GreaterThanOrEqual(attr, _) => isSupportedField(attr)
- case StringStartsWith(attr, _) => isSupportedField(attr)
- case IsNull(attr) => isSupportedField(attr)
- case IsNotNull(attr) => isSupportedField(attr)
- case Or(left, right) => isSupported(left) && isSupported(right)
- case And(left, right) => isSupported(left) && isSupported(right)
- case _ => false
- }
-
- def isSupportedField(attr: String): Boolean = {
- catalog.sMap.map.get(attr) match {
- case Some(field) => !(hasCompositeRowKey && field.isRowKey)
- case None => false
- }
- }
+ def dataTypeOf(attr: String): Option[DataType] =
+ catalog.sMap.map
+ .get(attr)
+ .filter(
+ field =>
+ !(hasCompositeRowKey && field.isRowKey) &&
+ field.avroSchema.isEmpty && field.serdes.isEmpty)
+ .map(_.dt)
val supported = new ListBuffer[Filter]()
val unsupported = new ListBuffer[Filter]()
- filters.foreach { f =>
- if (isSupported(f)) supported += f
- else unsupported += f
+ filters.foreach {
+ f =>
+ if (PredicatePushdownSupport.isSupported(
+ f,
+ dataTypeOf,
+ encoder.getClass == classOf[NaiveEncoder])) supported += f
Review Comment:
The exact-class check is intentional for this conservative fix: subclasses
can override encode, filter, or ranges.
I'll document that only the built-in NaiveEncoder is currently allowed for
range pushdown. Custom encoder range predicates remain residual. An explicit
opt-in capability API can be handled separately.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
