wchevreuil commented on code in PR #174:
URL: https://github.com/apache/hbase-connectors/pull/174#discussion_r4491552636
##########
spark4/hbase-spark4/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseScanBuilder.scala:
##########
@@ -57,33 +57,25 @@ class HBaseScanBuilder(schema: StructType, properties:
Map[String, String])
override def pushFilters(filters: Array[Filter]): Array[Filter] = {
val hasCompositeRowKey = catalog.getRowKey.size > 1
- def isSupported(f: Filter): Boolean = f match {
- case EqualTo(attr, _) => isSupportedField(attr)
- case LessThan(attr, _) => isSupportedField(attr)
- case GreaterThan(attr, _) => isSupportedField(attr)
- case LessThanOrEqual(attr, _) => isSupportedField(attr)
- case GreaterThanOrEqual(attr, _) => isSupportedField(attr)
- case StringStartsWith(attr, _) => isSupportedField(attr)
- case IsNull(attr) => isSupportedField(attr)
- case IsNotNull(attr) => isSupportedField(attr)
- case Or(left, right) => isSupported(left) && isSupported(right)
- case And(left, right) => isSupported(left) && isSupported(right)
- case _ => false
- }
-
- def isSupportedField(attr: String): Boolean = {
- catalog.sMap.map.get(attr) match {
- case Some(field) => !(hasCompositeRowKey && field.isRowKey)
- case None => false
- }
- }
+ def dataTypeOf(attr: String): Option[DataType] =
+ catalog.sMap.map
+ .get(attr)
+ .filter(
+ field =>
+ !(hasCompositeRowKey && field.isRowKey) &&
+ field.avroSchema.isEmpty && field.serdes.isEmpty)
Review Comment:
Thanks for clarifying and addressing it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
