andygrove commented on code in PR #391:
URL: https://github.com/apache/datafusion-spark/pull/391#discussion_r4646386857
##########
crates/sqllogictest/scripts/pyspark_slt_runner.py:
##########
@@ -149,12 +216,12 @@ def parse_slt(path: Path) -> list[Record]:
if words[0] not in ("statement", "query"):
# Directives such as `include` and `halt`, which these files do
not use.
conditions = []
- xfail_reason = None
+ xfails = []
i += 1
continue
- record = Record(path=path, line=i + 1, kind=words[0],
xfail_reason=xfail_reason)
- xfail_reason = None
+ record = Record(path=path, line=i + 1, kind=words[0], xfails=xfails)
Review Comment:
Done in f4b9128. A marker that no supported version reaches now stops the
runner at parse time. I changed `test_xfail_spark_versions` and the README
example to `spark>=4.0, <=4.2`, and added `test_xfail_that_never_applies` for
both the unreachable and the shadowed case.
##########
crates/sqllogictest/scripts/pyspark_slt_runner.py:
##########
@@ -788,6 +865,123 @@ def translate(sql: str, time_zone: str = "UTC") -> str:
return "".join(out)
+# Keywords after which a `-` is a sign, not a subtraction, and a `(` does
+# not start the arguments of a call.
+_KEYWORDS = {
+ "ALL", "AND", "ANY", "AS", "BETWEEN", "BY", "CASE", "DISTINCT", "ELSE",
+ "EXISTS", "FROM", "HAVING", "IN", "IS", "JOIN", "LIKE", "LIMIT", "NOT",
Review Comment:
Done in f4b9128. Both left the keyword set, and both cases are in
`test_double_colon_casts_for_spark_3`.
##########
crates/sqllogictest/scripts/pyspark_slt_runner.py:
##########
@@ -788,6 +865,123 @@ def translate(sql: str, time_zone: str = "UTC") -> str:
return "".join(out)
+# Keywords after which a `-` is a sign, not a subtraction, and a `(` does
+# not start the arguments of a call.
+_KEYWORDS = {
+ "ALL", "AND", "ANY", "AS", "BETWEEN", "BY", "CASE", "DISTINCT", "ELSE",
+ "EXISTS", "FROM", "HAVING", "IN", "IS", "JOIN", "LIKE", "LIMIT", "NOT",
+ "ON", "OR", "RETURN", "SELECT", "THEN", "VALUES", "WHEN", "WHERE",
+}
+
+
+# The type names of typed literals, such as DATE '2024-01-01'.
+_TYPED_LITERALS = ("DATE", "TIME", "INTERVAL", "X", *_TIMESTAMP_TYPES)
+
+# Brackets that nest in a type name, such as `array<decimal(10, 2)>`.
+_TYPE_NESTING = {"(": 1, "<": 1, ")": -1, ">": -1}
+
+_UNTRANSLATABLE = "a `::` cast this runner cannot translate"
+
+
+def rewrite_double_colons(sql: str) -> str:
+ """Replace each `expr::type` with `CAST(expr AS type)`.
+
+ Spark 3.5 does not parse `::`. As in Spark 4, `::` applies to the primary
+ expression before it: a literal, a column, a call, a parenthesized
+ expression or a subscript. A sign belongs to a number literal, so
+ `-1::int` is `CAST(-1 AS int)`, but `-(1)::int` is `-CAST((1) AS int)`.
+ """
+ tokens = _tokenize(sql)
+ out: list[tuple[str, str]] = []
+ i = 0
+ while i < len(tokens):
+ if tokens[i][0] != "cast":
+ out.append(tokens[i])
+ i += 1
+ continue
+ start = _operand_start(out)
+ operand = "".join(t for _, t in out[start:]).strip()
+ del out[start:]
+ end = _type_end(tokens, i + 1)
+ type_sql = "".join(t for _, t in tokens[i + 1 : end]).strip()
+ if not operand or not type_sql:
+ raise Skip(_UNTRANSLATABLE)
+ out.append(("operand", f"CAST({operand} AS {type_sql})"))
+ i = end
+ return "".join(t for _, t in out)
+
+
+def _significant_before(tokens: list[tuple[str, str]], end: int) -> int:
+ """The index of the last token before `end` that is not whitespace, or
-1."""
+ k = end - 1
+ while k >= 0 and tokens[k][0] in ("space", "comment"):
+ k -= 1
+ return k
+
+
+def _operand_start(tokens: list[tuple[str, str]]) -> int:
+ """The index where the primary expression that ends `tokens` starts."""
+ k = _significant_before(tokens, len(tokens))
+ if k < 0:
+ raise Skip(_UNTRANSLATABLE)
+ kind, text = tokens[k]
+ if kind == "other" and text in (")", "]"):
+ k = _matching_open(tokens, k)
+ before = _significant_before(tokens, k)
+ prev = tokens[before] if before >= 0 else ("", "")
+ if text == ")" and prev[0] == "word" and prev[1].upper() not in
_KEYWORDS:
+ # A call, f(...)
+ return _name_start(tokens, before)
+ if text == "]" and (prev[0] in ("word", "quoted", "string") or prev[1]
in (")", "]")):
+ # A subscript, a[...], of the operand that ends before it
+ return _operand_start(tokens[: before + 1])
+ return k
+ if kind == "string":
+ before = _significant_before(tokens, k)
+ if before >= 0 and tokens[before][0] == "word" and
tokens[before][1].upper() in _TYPED_LITERALS:
+ # A typed literal, such as DATE '2024-01-01'.
+ return before
+ return k
+ if kind == "number":
+ before = _significant_before(tokens, k)
+ if before >= 0 and tokens[before][1] == "-":
+ sign_of = _significant_before(tokens, before)
+ prev = tokens[sign_of] if sign_of >= 0 else None
+ if (
+ prev is None
+ or (prev[0] == "other" and prev[1] not in (")", "]"))
+ or (prev[0] == "word" and prev[1].upper() in _KEYWORDS)
+ ):
+ return before
+ return k
+ if kind in ("word", "quoted"):
+ return _name_start(tokens, k)
+ if kind == "operand":
+ # A cast that this function already rewrote, as in `x::int::string`.
+ return k
+ raise Skip(_UNTRANSLATABLE)
+
+
+def _name_start(tokens: list[tuple[str, str]], k: int) -> int:
+ """The start of a possibly qualified name, such as `t.a`, that ends at
tokens[k]."""
+ while k >= 2 and tokens[k - 1][1] == "." and tokens[k - 2][0] in ("word",
"quoted"):
+ k -= 2
+ return k
Review Comment:
Done in f4b9128, with tests for `struct(1 AS x).x::string` and `map(1,
struct(2 AS y))[1].y::string`.
##########
crates/sqllogictest/scripts/pyspark_slt_runner.py:
##########
@@ -788,6 +865,123 @@ def translate(sql: str, time_zone: str = "UTC") -> str:
return "".join(out)
+# Keywords after which a `-` is a sign, not a subtraction, and a `(` does
+# not start the arguments of a call.
+_KEYWORDS = {
+ "ALL", "AND", "ANY", "AS", "BETWEEN", "BY", "CASE", "DISTINCT", "ELSE",
+ "EXISTS", "FROM", "HAVING", "IN", "IS", "JOIN", "LIKE", "LIMIT", "NOT",
+ "ON", "OR", "RETURN", "SELECT", "THEN", "VALUES", "WHEN", "WHERE",
+}
+
+
+# The type names of typed literals, such as DATE '2024-01-01'.
+_TYPED_LITERALS = ("DATE", "TIME", "INTERVAL", "X", *_TIMESTAMP_TYPES)
+
+# Brackets that nest in a type name, such as `array<decimal(10, 2)>`.
+_TYPE_NESTING = {"(": 1, "<": 1, ")": -1, ">": -1}
+
+_UNTRANSLATABLE = "a `::` cast this runner cannot translate"
+
+
+def rewrite_double_colons(sql: str) -> str:
+ """Replace each `expr::type` with `CAST(expr AS type)`.
+
+ Spark 3.5 does not parse `::`. As in Spark 4, `::` applies to the primary
+ expression before it: a literal, a column, a call, a parenthesized
+ expression or a subscript. A sign belongs to a number literal, so
+ `-1::int` is `CAST(-1 AS int)`, but `-(1)::int` is `-CAST((1) AS int)`.
+ """
+ tokens = _tokenize(sql)
+ out: list[tuple[str, str]] = []
+ i = 0
+ while i < len(tokens):
+ if tokens[i][0] != "cast":
+ out.append(tokens[i])
+ i += 1
+ continue
+ start = _operand_start(out)
+ operand = "".join(t for _, t in out[start:]).strip()
+ del out[start:]
+ end = _type_end(tokens, i + 1)
+ type_sql = "".join(t for _, t in tokens[i + 1 : end]).strip()
+ if not operand or not type_sql:
+ raise Skip(_UNTRANSLATABLE)
+ out.append(("operand", f"CAST({operand} AS {type_sql})"))
+ i = end
+ return "".join(t for _, t in out)
+
+
+def _significant_before(tokens: list[tuple[str, str]], end: int) -> int:
+ """The index of the last token before `end` that is not whitespace, or
-1."""
+ k = end - 1
+ while k >= 0 and tokens[k][0] in ("space", "comment"):
+ k -= 1
+ return k
+
+
+def _operand_start(tokens: list[tuple[str, str]]) -> int:
+ """The index where the primary expression that ends `tokens` starts."""
+ k = _significant_before(tokens, len(tokens))
+ if k < 0:
+ raise Skip(_UNTRANSLATABLE)
+ kind, text = tokens[k]
+ if kind == "other" and text in (")", "]"):
+ k = _matching_open(tokens, k)
+ before = _significant_before(tokens, k)
+ prev = tokens[before] if before >= 0 else ("", "")
+ if text == ")" and prev[0] == "word" and prev[1].upper() not in
_KEYWORDS:
+ # A call, f(...)
+ return _name_start(tokens, before)
+ if text == "]" and (prev[0] in ("word", "quoted", "string") or prev[1]
in (")", "]")):
+ # A subscript, a[...], of the operand that ends before it
+ return _operand_start(tokens[: before + 1])
+ return k
+ if kind == "string":
+ before = _significant_before(tokens, k)
+ if before >= 0 and tokens[before][0] == "word" and
tokens[before][1].upper() in _TYPED_LITERALS:
+ # A typed literal, such as DATE '2024-01-01'.
+ return before
+ return k
+ if kind == "number":
+ before = _significant_before(tokens, k)
+ if before >= 0 and tokens[before][1] == "-":
+ sign_of = _significant_before(tokens, before)
+ prev = tokens[sign_of] if sign_of >= 0 else None
+ if (
+ prev is None
+ or (prev[0] == "other" and prev[1] not in (")", "]"))
+ or (prev[0] == "word" and prev[1].upper() in _KEYWORDS)
+ ):
+ return before
+ return k
+ if kind in ("word", "quoted"):
+ return _name_start(tokens, k)
+ if kind == "operand":
+ # A cast that this function already rewrote, as in `x::int::string`.
+ return k
+ raise Skip(_UNTRANSLATABLE)
+
+
+def _name_start(tokens: list[tuple[str, str]], k: int) -> int:
+ """The start of a possibly qualified name, such as `t.a`, that ends at
tokens[k]."""
+ while k >= 2 and tokens[k - 1][1] == "." and tokens[k - 2][0] in ("word",
"quoted"):
+ k -= 2
+ return k
+
+
+def _type_end(tokens: list[tuple[str, str]], start: int) -> int:
+ """The index after the type name that starts at tokens[start], such as
+ `int`, `decimal(10, 2)` or `array<map<string, int>>`."""
+ k = _next_significant(tokens, start)
+ if k >= len(tokens) or tokens[k][0] != "word":
+ raise Skip(_UNTRANSLATABLE)
+ k += 1
+ j = _next_significant(tokens, k)
+ if j < len(tokens) and tokens[j][1] in ("(", "<"):
Review Comment:
Done in f4b9128. Only `ARRAY`, `MAP` and `STRUCT` read `<` as type
parameters now. I added `a::int < 5 AND b > 3` and `a::int<b::int` to the tests.
##########
crates/sqllogictest/test_files/math/greatest.slt:
##########
@@ -93,11 +93,13 @@ SELECT arrow_typeof(greatest(1::decimal(38,10),
1::decimal(38,0)));
Decimal128(38, 0)
# The fractional digits that do not fit are rounded away
+# xfail spark<4.0: intended: Spark 3.5 picks DECIMAL(38, 10) as the wider type
and returns NULL for the value that does not fit, and datafusion-spark follows
Spark 4
Review Comment:
Thanks, applied in f4b9128.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]