dbatomic commented on code in PR #46801:
URL: https://github.com/apache/spark/pull/46801#discussion_r1625996529
##########
sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala:
##########
@@ -948,6 +952,210 @@ class CollationSuite extends DatasourceV2SQLBase with
AdaptiveSparkPlanHelper {
}
}
+ test("SPARK-48280: Expression Walker for Testing") {
+ // This test does following:
+ // 1) Take all expressions
+ // 2) Filter out ones that have at least one argument of StringType
+ // 3) Use reflection to create an instance of the expression using first
constructor
+ // (test other as well).
+ // 4) Check if the expression is of type ExpectsInputTypes (should make
this a bit broader)
+ // 5) Run eval against literals with strings under:
+ // a) UTF8_BINARY, "dummy string" as input.
+ // b) UTF8_BINARY_LCASE, "DuMmY sTrInG" as input.
+ // 6) Check if both expressions throw an exception.
+ // 7) If no exception, check if the result is the same.
+ // 8) There is a list of allowed expressions that can differ (e.g. hex)
+ def hasStringType(inputType: AbstractDataType): Boolean = {
+ inputType match {
+ case _: StringType | StringTypeAnyCollation | StringTypeBinaryLcase |
AnyDataType =>
+ true
+ case ArrayType => true
+ case ArrayType(elementType, _) => hasStringType(elementType)
+ case AbstractArrayType(elementType) => hasStringType(elementType)
+ case TypeCollection(typeCollection) =>
+ typeCollection.exists(hasStringType)
+ case _ => false
+ }
+ }
+
+ val funInfos = spark.sessionState.functionRegistry.listFunction().map {
funcId =>
+ spark.sessionState.catalog.lookupFunctionInfo(funcId)
+ }.filter(funInfo => {
+ // make sure that there is a constructor.
+ val cl = Utils.classForName(funInfo.getClassName)
+ !cl.getConstructors.isEmpty
+ }).filter(funInfo => {
+ val className = funInfo.getClassName
+ val cl = Utils.classForName(funInfo.getClassName)
+ // dummy instance
+ // Take first constructor.
+ val headConstructor = cl.getConstructors.head
+
+ val params = headConstructor.getParameters.map(p => p.getType)
+ val allExpressions = params.forall(p =>
p.isAssignableFrom(classOf[Expression]) ||
+ p.isAssignableFrom(classOf[Seq[Expression]]) ||
+ p.isAssignableFrom(classOf[Option[Expression]]))
+
+ if (!allExpressions) {
+ false
+ } else {
+ val args = params.map {
+ case e if e.isAssignableFrom(classOf[Expression]) =>
Literal.create("1")
+ case se if se.isAssignableFrom(classOf[Seq[Expression]]) =>
+ Seq(Literal.create("1"), Literal.create("2"))
+ case oe if oe.isAssignableFrom(classOf[Option[Expression]]) => None
+ }
+ // Find all expressions that have string as input
+ try {
+ val expr = headConstructor.newInstance(args: _*)
+ expr match {
+ case types: ExpectsInputTypes =>
+ val inputTypes = types.inputTypes
+ // check if this is a collection...
+ inputTypes.exists(hasStringType)
+ }
+ } catch {
+ case _: Throwable => false
+ }
+ }
+ }).toArray
+
+ // Helper methods for generating data.
+ sealed trait CollationType
+ case object Utf8Binary extends CollationType
+ case object Utf8BinaryLcase extends CollationType
+
+ def generateSingleEntry(
+ inputType: AbstractDataType,
+ collationType: CollationType): Expression =
+ inputType match {
+ // Try to make this a bit more random.
+ case AnyTimestampType => Literal("2009-07-30 12:58:59")
Review Comment:
Let's try to organize this a bit better. I think that in future this logic
may become more complex (e.g. we don't want to just pass 1 and "dummy_string".
Instead we will try with different string shapes + special rules for integers
(-1, 0, 1, strlen, strlen + 1...).
Again, my recommendation is to add new class for expression walker and
define this logic as methods of that class.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
