srielau commented on code in PR #52638:
URL: https://github.com/apache/spark/pull/52638#discussion_r2465725879
##########
sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/DataTypeAstBuilder.scala:
##########
@@ -399,3 +569,54 @@ class DataTypeAstBuilder extends
SqlBaseParserBaseVisitor[AnyRef] {
(start.getOrElse(defaultStart), step.getOrElse(defaultStep))
}
}
+
+/**
+ * A synthetic token representing multiple coalesced string literals.
+ *
+ * When the parser encounters consecutive string literals (e.g., 'hello'
'world'), they are
+ * automatically coalesced into a single logical string. This token class
represents such
+ * coalesced strings while maintaining the Token interface.
+ *
+ * The coalescedValue contains the raw concatenated content from all the
string literals (with
+ * outer quotes removed but escape sequences preserved). The getText() method
wraps this in
+ * quotes, and when SparkParserUtils.string() is called, it will unescape the
content based on the
+ * current SQL configuration (respecting ESCAPED_STRING_LITERALS).
+ *
+ * @param source
+ * The token source and input stream
+ * @param tokenType
+ * The ANTLR token type (typically STRING_LITERAL)
+ * @param channel
+ * The token channel
+ * @param start
+ * The start index of the first literal in the input stream
+ * @param stop
+ * The stop index of the last literal in the input stream
+ * @param coalescedValue
+ * The raw concatenated content (without outer quotes, escape sequences NOT
processed)
+ */
+private[parser] class CoalescedStringToken(
+ source: Pair[TokenSource, CharStream],
+ tokenType: Int,
+ channel: Int,
+ start: Int,
+ stop: Int,
+ private val coalescedValue: String,
+ private val isRawString: Boolean = false,
+ private val quoteChar: Char = '\'')
+ extends CommonToken(source, tokenType, channel, start, stop) {
+
+ override def getText: String = {
+ if (isRawString) {
Review Comment:
Yes, in master each token is processed individual. Fixing.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
