Sigma-Ma commented on code in PR #174:
URL: https://github.com/apache/hbase-connectors/pull/174#discussion_r4488899063
##########
spark4/hbase-spark4/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseScanBuilder.scala:
##########
@@ -57,33 +57,25 @@ class HBaseScanBuilder(schema: StructType, properties:
Map[String, String])
override def pushFilters(filters: Array[Filter]): Array[Filter] = {
val hasCompositeRowKey = catalog.getRowKey.size > 1
- def isSupported(f: Filter): Boolean = f match {
- case EqualTo(attr, _) => isSupportedField(attr)
- case LessThan(attr, _) => isSupportedField(attr)
- case GreaterThan(attr, _) => isSupportedField(attr)
- case LessThanOrEqual(attr, _) => isSupportedField(attr)
- case GreaterThanOrEqual(attr, _) => isSupportedField(attr)
- case StringStartsWith(attr, _) => isSupportedField(attr)
- case IsNull(attr) => isSupportedField(attr)
- case IsNotNull(attr) => isSupportedField(attr)
- case Or(left, right) => isSupported(left) && isSupported(right)
- case And(left, right) => isSupported(left) && isSupported(right)
- case _ => false
- }
-
- def isSupportedField(attr: String): Boolean = {
- catalog.sMap.map.get(attr) match {
- case Some(field) => !(hasCompositeRowKey && field.isRowKey)
- case None => false
- }
- }
+ def dataTypeOf(attr: String): Option[DataType] =
+ catalog.sMap.map
+ .get(attr)
+ .filter(
+ field =>
+ !(hasCompositeRowKey && field.isRowKey) &&
+ field.avroSchema.isEmpty && field.serdes.isEmpty)
Review Comment:
That check prevents pushdown, but it also makes Spark request the Avro field
for residual evaluation.
For an Avro int field, SELECT key WHERE age = 100 worked before this change
because age was filtered without being decoded. After fallback,
AvroSerdes.deserialize attempts to return the primitive as a GenericRecord and
throws ClassCastException. I reproduced this against the PR head.
I'll address the fallback read path and add this query as an end-to-end
regression.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
