zhengruifeng commented on code in PR #39016:
URL: https://github.com/apache/spark/pull/39016#discussion_r1045163182
##########
python/pyspark/sql/connect/column.py:
##########
@@ -180,7 +186,12 @@ def to_plan(self, session: "SparkConnectClient") ->
"proto.Expression":
elif isinstance(self._value, bool):
expr.literal.boolean = bool(self._value)
elif isinstance(self._value, int):
- expr.literal.long = int(self._value)
+ if JVM_INT_MIN <= self._value <= JVM_INT_MAX:
Review Comment:
```
In [1]: from pyspark.sql.connect.column import JVM_INT_MIN, JVM_INT_MAX,
JVM_LONG_MIN, JVM_LONG_MAX
In [2]: sdf = spark.range(0, 1)
In [3]: sdf.select(lit(0), lit(JVM_LONG_MAX + 1))
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[3], line 1
----> 1 sdf.select(lit(0), lit(JVM_LONG_MAX + 1))
NameError: name 'lit' is not defined
In [4]: from pyspark.sql.functions import *
In [5]: sdf.select(lit(0), lit(JVM_LONG_MAX + 1))
---------------------------------------------------------------------------
Py4JError Traceback (most recent call last)
Cell In[5], line 1
----> 1 sdf.select(lit(0), lit(JVM_LONG_MAX + 1))
File ~/Dev/spark/python/pyspark/sql/functions.py:179, in lit(col)
177 if dt is not None:
178 return _invoke_function("lit", col).astype(dt)
--> 179 return _invoke_function("lit", col)
File ~/Dev/spark/python/pyspark/sql/functions.py:88, in
_invoke_function(name, *args)
86 assert SparkContext._active_spark_context is not None
87 jf = _get_jvm_function(name, SparkContext._active_spark_context)
---> 88 return Column(jf(*args))
File ~/Dev/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/java_gateway.py:1322,
in JavaMember.__call__(self, *args)
1316 command = proto.CALL_COMMAND_NAME +\
1317 self.command_header +\
1318 args_command +\
1319 proto.END_COMMAND_PART
1321 answer = self.gateway_client.send_command(command)
-> 1322 return_value = get_return_value(
1323 answer, self.gateway_client, self.target_id, self.name)
1325 for temp_arg in temp_args:
1326 if hasattr(temp_arg, "_detach"):
File ~/Dev/spark/python/pyspark/sql/utils.py:199, in
capture_sql_exception.<locals>.deco(*a, **kw)
197 def deco(*a: Any, **kw: Any) -> Any:
198 try:
--> 199 return f(*a, **kw)
200 except Py4JJavaError as e:
201 converted = convert_exception(e.java_exception)
File ~/Dev/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/protocol.py:330, in
get_return_value(answer, gateway_client, target_id, name)
326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
328 format(target_id, ".", name), value)
329 else:
--> 330 raise Py4JError(
331 "An error occurred while calling {0}{1}{2}.
Trace:\n{3}\n".
332 format(target_id, ".", name, value))
333 else:
334 raise Py4JError(
335 "An error occurred while calling {0}{1}{2}".
336 format(target_id, ".", name))
Py4JError: An error occurred while calling
z:org.apache.spark.sql.functions.lit. Trace:
java.lang.NumberFormatException: For input string: "9223372036854775808"
at
java.base/java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
at java.base/java.lang.Long.parseLong(Long.java:696)
at java.base/java.lang.Long.parseLong(Long.java:817)
at py4j.Protocol.getLong(Protocol.java:239)
at py4j.Protocol.getObject(Protocol.java:291)
at py4j.commands.AbstractCommand.getArguments(AbstractCommand.java:82)
at py4j.commands.CallCommand.execute(CallCommand.java:77)
at
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.base/java.lang.Thread.run(Thread.java:829)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
