cloud-fan commented on a change in pull request #34001:
URL: https://github.com/apache/spark/pull/34001#discussion_r709768914
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
##########
@@ -261,4 +267,116 @@ object DataSourceUtils extends PredicateHelper {
dataFilters.flatMap(extractPredicatesWithinOutputSet(_, partitionSet))
(ExpressionSet(partitionFilters ++ extraPartitionFilter).toSeq,
dataFilters)
}
+
+ def convertV1FilterToV2(v1Filter: sources.Filter): V2Filter = {
+ v1Filter match {
+ case _: sources.AlwaysFalse =>
+ new V2AlwaysFalse
+ case _: sources.AlwaysTrue =>
+ new V2AlwaysTrue
+ case e: sources.EqualNullSafe =>
+ new V2EqualNullSafe(FieldReference(e.attribute),
getLiteralValue(e.value))
+ case equal: sources.EqualTo =>
+ new V2EqualTo(FieldReference(equal.attribute),
getLiteralValue(equal.value))
+ case g: sources.GreaterThan =>
+ new V2GreaterThan(FieldReference(g.attribute),
getLiteralValue(g.value))
+ case ge: sources.GreaterThanOrEqual =>
+ new V2GreaterThanOrEqual(FieldReference(ge.attribute),
getLiteralValue(ge.value))
+ case in: sources.In =>
+ new V2In(FieldReference(
+ in.attribute), in.values.map(value => getLiteralValue(value)))
+ case notNull: sources.IsNotNull =>
+ new V2IsNotNull(FieldReference(notNull.attribute))
+ case isNull: sources.IsNull =>
+ new V2IsNull(FieldReference(isNull.attribute))
+ case l: sources.LessThan =>
+ new V2LessThan(FieldReference(l.attribute), getLiteralValue(l.value))
+ case le: sources.LessThanOrEqual =>
+ new V2LessThanOrEqual(FieldReference(le.attribute),
getLiteralValue(le.value))
+ case contains: sources.StringContains =>
+ new V2StringContains(
+ FieldReference(contains.attribute),
UTF8String.fromString(contains.value))
+ case ends: sources.StringEndsWith =>
+ new V2StringEndsWith(FieldReference(ends.attribute),
UTF8String.fromString(ends.value))
+ case starts: sources.StringStartsWith =>
+ new V2StringStartsWith(
+ FieldReference(starts.attribute),
UTF8String.fromString(starts.value))
+ case and: sources.And =>
+ new V2And(convertV1FilterToV2(and.left),
convertV1FilterToV2(and.right))
+ case or: sources.Or =>
+ new V2Or(convertV1FilterToV2(or.left), convertV1FilterToV2(or.right))
+ case not: sources.Not =>
+ new V2Not(convertV1FilterToV2(not.child))
+ case _ => throw new IllegalStateException("Invalid v1Filter: " +
v1Filter)
+ }
+ }
+
+ def getLiteralValue(value: Any): LiteralValue[_] = value match {
+ case _: JavaBigDecimal =>
+ LiteralValue(Decimal(value.asInstanceOf[JavaBigDecimal]),
DecimalType.SYSTEM_DEFAULT)
+ case _: JavaBigInteger =>
+ LiteralValue(Decimal(value.asInstanceOf[JavaBigInteger]),
DecimalType.SYSTEM_DEFAULT)
+ case _: BigDecimal =>
+ LiteralValue(Decimal(value.asInstanceOf[BigDecimal]),
DecimalType.SYSTEM_DEFAULT)
+ case _: Boolean => LiteralValue(value, BooleanType)
+ case _: Byte => LiteralValue(value, ByteType)
+ case _: Array[Byte] => LiteralValue(value, BinaryType)
+ case _: Date =>
+ val date = DateTimeUtils.fromJavaDate(value.asInstanceOf[Date])
+ LiteralValue(date, DateType)
+ case _: LocalDate =>
+ val date = DateTimeUtils.localDateToDays(value.asInstanceOf[LocalDate])
+ LiteralValue(date, DateType)
+ case _: Double => LiteralValue(value, DoubleType)
+ case _: Float => LiteralValue(value, FloatType)
+ case _: Integer => LiteralValue(value, IntegerType)
+ case _: Long => LiteralValue(value, LongType)
+ case _: Short => LiteralValue(value, ShortType)
+ case _: String => LiteralValue(UTF8String.fromString(value.toString),
StringType)
+ case _: Timestamp =>
+ val ts = DateTimeUtils.fromJavaTimestamp(value.asInstanceOf[Timestamp])
+ LiteralValue(ts, TimestampType)
+ case _: Instant =>
+ val ts = DateTimeUtils.instantToMicros(value.asInstanceOf[Instant])
+ LiteralValue(ts, TimestampType)
+ case _ =>
Review comment:
how about array/map/struct?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
