jubins commented on code in PR #59214:
URL: https://github.com/apache/spark/pull/59214#discussion_r4171447562
##########
python/pyspark/sql/variant_utils.py:
##########
@@ -413,10 +417,81 @@ def handle_array(value_pos_list: List[int]) -> str:
if isinstance(value, bytes):
# decoding simply converts byte array to string
return '"' + base64.b64encode(value).decode("utf-8") + '"'
- if isinstance(value, (datetime.date, datetime.datetime)):
+ if isinstance(value, float):
+ return cls._float_to_json(value, is_float32)
+ if isinstance(value, decimal.Decimal):
+ return cls._decimal_to_json(value)
+ if isinstance(value, datetime.datetime):
+ return '"' + cls._timestamp_to_json(value) + '"'
+ if isinstance(value, datetime.date):
return '"' + str(value) + '"'
return str(value)
+ # The helpers below format scalars the same way as the JVM
`Variant.toJson`, so that
+ # `VariantVal.toJson` and the `to_json` SQL function produce the same
output.
+
+ @classmethod
+ def _float_to_json(cls, f: float, is_float32: bool) -> str:
+ """
+ Formats a double or float like Java's `Double.toString` and
`Float.toString`. Non-finite
+ values are quoted because they are not valid JSON numbers.
+ """
+ if math.isnan(f):
+ return '"NaN"'
+ if math.isinf(f):
+ return '"Infinity"' if f > 0 else '"-Infinity"'
+
+ def round_trips(s: str) -> bool:
+ if not is_float32:
+ return float(s) == f
+ try:
+ return struct.pack("<f", float(s)) == struct.pack("<f", f)
+ except OverflowError:
+ return False
+
+ # Like Java, pick the shortest correctly rounded decimal that
round-trips, using at least
+ # 2 significant digits (e.g. Double.MIN_VALUE is "4.9E-324", not
"5.0E-324").
+ for precision in range(2, 18):
Review Comment:
@HyukjinKwon thanks for the careful check! Fixed if the closest decimal
doesn't round-trip, it now tries the next one away from zero before adding a
digit. All double and float powers of two now match `to_json`, and I added
`2^-24`, `2^-44` and `2^-96` as test cases.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
