jubins commented on code in PR #59214:
URL: https://github.com/apache/spark/pull/59214#discussion_r4362853159
##########
python/pyspark/sql/tests/test_types.py:
##########
@@ -3864,6 +3864,83 @@ def test_struct_field_type_name(self):
struct_field = StructField("a", IntegerType())
self.assertRaises(TypeError, struct_field.typeName)
+ def test_variant_to_json_matches_jvm_format(self):
+ import decimal
+ import struct
+
+ def variant(type_info, payload):
+ return VariantVal(bytes([type_info << 2]) + payload, bytes([1, 0,
0]))
+
+ def double(x):
+ return variant(7, struct.pack("<d", x))
+
+ def float32(x):
+ return variant(14, struct.pack("<f", x))
+
+ def decimal4(unscaled, scale):
+ return variant(8, bytes([scale]) + struct.pack("<i", unscaled))
+
+ def timestamp(micros, ntz):
+ return variant(13 if ntz else 12, struct.pack("<q", micros))
+
+ # The expected values are what the JVM `to_json` returns for the same
variants on JDK 19+.
+ # On older JDKs, `to_json` can print longer digit strings for some
doubles and floats,
+ # e.g. the powers of two below.
+ for v, expected in [
+ (double(float("nan")), '"NaN"'),
+ (double(float("inf")), '"Infinity"'),
+ (double(float("-inf")), '"-Infinity"'),
+ (double(0.0), "0.0"),
+ (double(-0.0), "-0.0"),
+ (double(123.0), "123.0"),
+ (double(123456.789), "123456.789"),
+ (double(0.001), "0.001"),
+ (double(1e7), "1.0E7"),
+ (double(1.5e-5), "1.5E-5"),
+ (double(-1e20), "-1.0E20"),
+ (double(5e-324), "4.9E-324"),
+ (double(0.1 + 0.2), "0.30000000000000004"),
+ # Powers of two whose shortest decimal is not the closest one of
that length.
+ (double(2.0**-24), "5.960464477539063E-8"),
+ (double(-(2.0**-24)), "-5.960464477539063E-8"),
+ (double(2.0**-44), "5.684341886080802E-14"),
+ (float32(2.0**-96), "1.2621775E-29"),
+ (float32(-(2.0**-96)), "-1.2621775E-29"),
+ # Neighboring floats whose shortest decimals, read as doubles, are
exactly the
Review Comment:
Reworded: both floats share the closest 7-digit decimal 7.038531E-26, which
is exactly the midpoint and names the lower float, so the upper one needs 8
digits.
##########
python/pyspark/sql/variant_utils.py:
##########
@@ -413,10 +417,130 @@ def handle_array(value_pos_list: List[int]) -> str:
if isinstance(value, bytes):
# decoding simply converts byte array to string
return '"' + base64.b64encode(value).decode("utf-8") + '"'
- if isinstance(value, (datetime.date, datetime.datetime)):
+ if isinstance(value, float):
+ return cls._float_to_json(value, is_float32)
+ if isinstance(value, decimal.Decimal):
+ return cls._decimal_to_json(value)
+ if isinstance(value, datetime.datetime):
+ return '"' + cls._timestamp_to_json(value) + '"'
+ if isinstance(value, datetime.date):
return '"' + str(value) + '"'
return str(value)
+ # The helpers below format scalars the same way as the JVM
`Variant.toJson` on JDK 19+, so
+ # that `VariantVal.toJson` and the `to_json` SQL function produce the same
output. On older
+ # JDKs, `to_json` can print a longer, still round-tripping digit string
for some doubles and
Review Comment:
Updated all three to say older JDKs can print a different, still
round-tripping digit string (sometimes longer).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
