srielau commented on code in PR #58549:
URL: https://github.com/apache/spark/pull/58549#discussion_r4027707917
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala:
##########
@@ -33,6 +33,28 @@ object CharVarcharUtils extends Logging with
SparkCharVarcharUtils {
// visible for testing
private[sql] val CHAR_VARCHAR_TYPE_STRING_METADATA_KEY =
"__CHAR_VARCHAR_TYPE_STRING"
+ /**
+ * Replaces CHAR/VARCHAR with their unconstrained string representation
regardless of session
+ * configuration. Use this only at physical boundaries, such as Arrow, that
encode all character
+ * string types as UTF8.
+ */
+ private[sql] def replaceCharVarcharWithStringForPhysicalType(dt: DataType):
DataType = dt match {
+ case ArrayType(elementType, containsNull) =>
+ ArrayType(replaceCharVarcharWithStringForPhysicalType(elementType),
containsNull)
+ case MapType(keyType, valueType, valueContainsNull) =>
+ MapType(
+ replaceCharVarcharWithStringForPhysicalType(keyType),
+ replaceCharVarcharWithStringForPhysicalType(valueType),
+ valueContainsNull)
+ case StructType(fields) =>
+ StructType(fields.map { field =>
+ field.copy(dataType =
replaceCharVarcharWithStringForPhysicalType(field.dataType))
+ })
+ case c: CharType => c.toStringType
+ case v: VarcharType => v.toStringType
+ case other => other
Review Comment:
Done — 9c2c7e61a08. physicalTypeHasCharVarchar and physical CHAR/VARCHAR
replacement now descend into UserDefinedType.sqlType, and the Arrow/UDF/UDTF
guards use that physical check. Added UDT storage fixtures in the Arrow UDF and
Python UDTF tests.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala:
##########
@@ -108,7 +108,22 @@ case class UserDefinedPythonFunction(
}
PythonAggregate(name, func, dataType, e, udfDeterministic, bufferStruct)
} else {
- PythonUDF(name, func, dataType, e, pythonEvalType, udfDeterministic)
+ val applyCharVarcharChecks =
+ CharVarcharUtils.shouldApplyWriteSideLengthCheck(SQLConf.get)
+ val resolvedDataType = if (applyCharVarcharChecks) {
Review Comment:
Done — 9c2c7e61a08. PythonUDF now exposes first-class CHAR/VARCHAR only when
charVarcharFirstClassTypes is true, independently of
shouldApplyWriteSideLengthCheck. Default policy keeps annotated STRING while
still applying write-side length checks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
