dbatomic commented on code in PR #46801:
URL: https://github.com/apache/spark/pull/46801#discussion_r1625982455
##########
sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala:
##########
@@ -948,6 +952,210 @@ class CollationSuite extends DatasourceV2SQLBase with
AdaptiveSparkPlanHelper {
}
}
+ test("SPARK-48280: Expression Walker for Testing") {
+ // This test does following:
+ // 1) Take all expressions
+ // 2) Filter out ones that have at least one argument of StringType
+ // 3) Use reflection to create an instance of the expression using first
constructor
+ // (test other as well).
+ // 4) Check if the expression is of type ExpectsInputTypes (should make
this a bit broader)
+ // 5) Run eval against literals with strings under:
+ // a) UTF8_BINARY, "dummy string" as input.
+ // b) UTF8_BINARY_LCASE, "DuMmY sTrInG" as input.
+ // 6) Check if both expressions throw an exception.
+ // 7) If no exception, check if the result is the same.
+ // 8) There is a list of allowed expressions that can differ (e.g. hex)
+ def hasStringType(inputType: AbstractDataType): Boolean = {
+ inputType match {
+ case _: StringType | StringTypeAnyCollation | StringTypeBinaryLcase |
AnyDataType =>
+ true
+ case ArrayType => true
+ case ArrayType(elementType, _) => hasStringType(elementType)
+ case AbstractArrayType(elementType) => hasStringType(elementType)
+ case TypeCollection(typeCollection) =>
+ typeCollection.exists(hasStringType)
+ case _ => false
+ }
+ }
+
+ val funInfos = spark.sessionState.functionRegistry.listFunction().map {
funcId =>
+ spark.sessionState.catalog.lookupFunctionInfo(funcId)
+ }.filter(funInfo => {
+ // make sure that there is a constructor.
+ val cl = Utils.classForName(funInfo.getClassName)
+ !cl.getConstructors.isEmpty
+ }).filter(funInfo => {
+ val className = funInfo.getClassName
+ val cl = Utils.classForName(funInfo.getClassName)
+ // dummy instance
+ // Take first constructor.
+ val headConstructor = cl.getConstructors.head
+
+ val params = headConstructor.getParameters.map(p => p.getType)
+ val allExpressions = params.forall(p =>
p.isAssignableFrom(classOf[Expression]) ||
+ p.isAssignableFrom(classOf[Seq[Expression]]) ||
+ p.isAssignableFrom(classOf[Option[Expression]]))
+
+ if (!allExpressions) {
+ false
+ } else {
+ val args = params.map {
+ case e if e.isAssignableFrom(classOf[Expression]) =>
Literal.create("1")
+ case se if se.isAssignableFrom(classOf[Seq[Expression]]) =>
+ Seq(Literal.create("1"), Literal.create("2"))
+ case oe if oe.isAssignableFrom(classOf[Option[Expression]]) => None
+ }
+ // Find all expressions that have string as input
+ try {
+ val expr = headConstructor.newInstance(args: _*)
+ expr match {
+ case types: ExpectsInputTypes =>
+ val inputTypes = types.inputTypes
+ // check if this is a collection...
+ inputTypes.exists(hasStringType)
+ }
+ } catch {
+ case _: Throwable => false
+ }
+ }
+ }).toArray
+
+ // Helper methods for generating data.
+ sealed trait CollationType
+ case object Utf8Binary extends CollationType
+ case object Utf8BinaryLcase extends CollationType
+
+ def generateSingleEntry(
+ inputType: AbstractDataType,
+ collationType: CollationType): Expression =
+ inputType match {
+ // Try to make this a bit more random.
+ case AnyTimestampType => Literal("2009-07-30 12:58:59")
+ case BinaryType => Literal(new Array[Byte](5))
+ case BooleanType => Literal(true)
+ case _: DatetimeType => Literal(1L)
+ case _: DecimalType => Literal(new Decimal)
+ case IntegerType | NumericType => Literal(1)
+ case LongType => Literal(1L)
+ case _: StringType | StringTypeAnyCollation | StringTypeBinaryLcase |
AnyDataType =>
+ collationType match {
+ case Utf8Binary =>
+ Literal.create("dummy string", StringType("UTF8_BINARY"))
+ case Utf8BinaryLcase =>
+ Literal.create("DuMmY sTrInG", StringType("UTF8_BINARY_LCASE"))
+ }
+ case TypeCollection(typeCollection) =>
+ val strTypes = typeCollection.filter(hasStringType)
+ if (strTypes.isEmpty) {
+ // Take first type
+ generateSingleEntry(typeCollection.head, collationType)
+ } else {
+ // Take first string type
+ generateSingleEntry(strTypes.head, collationType)
+ }
+ case AbstractArrayType(elementType) =>
+ generateSingleEntry(elementType, collationType).map(
+ lit => Literal.create(Seq(lit.asInstanceOf[Literal].value),
ArrayType(lit.dataType))
+ ).head
+ case ArrayType(elementType, _) =>
+ generateSingleEntry(elementType, collationType).map(
+ lit => Literal.create(Seq(lit.asInstanceOf[Literal].value),
ArrayType(lit.dataType))
+ ).head
+ case ArrayType =>
+ generateSingleEntry(StringTypeAnyCollation, collationType).map(
+ lit => Literal.create(Seq(lit.asInstanceOf[Literal].value),
ArrayType(lit.dataType))
+ ).head
+ }
+
+ def generateData(
+ inputTypes: Seq[AbstractDataType],
+ collationType: CollationType): Seq[Expression] = {
+ inputTypes.map(generateSingleEntry(_, collationType))
+ }
+
+ val toSkip = List(
+ "get_json_object",
+ "map_zip_with",
+ "printf",
+ "transform_keys",
+ "concat_ws",
+ "format_string",
+ "session_window",
+ "transform_values",
+ "arrays_zip",
+ "hex" // this is fine
Review Comment:
Let's add comment next to each expression why it is fine to skip it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
