jubins commented on code in PR #59214:
URL: https://github.com/apache/spark/pull/59214#discussion_r4247110592
##########
python/pyspark/sql/tests/test_types.py:
##########
@@ -3864,6 +3864,70 @@ def test_struct_field_type_name(self):
struct_field = StructField("a", IntegerType())
self.assertRaises(TypeError, struct_field.typeName)
+ def test_variant_to_json_matches_jvm_format(self):
+ import decimal
+ import struct
+
+ def variant(type_info, payload):
+ return VariantVal(bytes([type_info << 2]) + payload, bytes([1, 0,
0]))
+
+ def double(x):
+ return variant(7, struct.pack("<d", x))
+
+ def float32(x):
+ return variant(14, struct.pack("<f", x))
+
+ def decimal4(unscaled, scale):
+ return variant(8, bytes([scale]) + struct.pack("<i", unscaled))
+
+ def timestamp(micros, ntz):
+ return variant(13 if ntz else 12, struct.pack("<q", micros))
+
+ # The expected values are what the JVM `to_json` returns for the same
variants on JDK 19+.
+ # On older JDKs, `to_json` can print longer digit strings for some
doubles and floats,
+ # e.g. the powers of two below.
+ for v, expected in [
+ (double(float("nan")), '"NaN"'),
+ (double(float("inf")), '"Infinity"'),
+ (double(float("-inf")), '"-Infinity"'),
+ (double(0.0), "0.0"),
+ (double(-0.0), "-0.0"),
+ (double(123.0), "123.0"),
+ (double(123456.789), "123456.789"),
+ (double(0.001), "0.001"),
+ (double(1e7), "1.0E7"),
+ (double(1.5e-5), "1.5E-5"),
+ (double(-1e20), "-1.0E20"),
+ (double(5e-324), "4.9E-324"),
+ # Powers of two whose shortest decimal is not the closest one of
that length.
+ (double(2.0**-24), "5.960464477539063E-8"),
+ (double(-(2.0**-24)), "-5.960464477539063E-8"),
+ (double(2.0**-44), "5.684341886080802E-14"),
+ (float32(2.0**-96), "1.2621775E-29"),
+ (float32(-(2.0**-96)), "-1.2621775E-29"),
+ (float32(float("nan")), '"NaN"'),
+ (float32(1.1), "1.1"),
+ (float32(1e10), "1.0E10"),
+ (float32(3.4028234663852886e38), "3.4028235E38"),
+ (decimal4(100, 2), "1"),
+ (decimal4(0, 2), "0"),
+ (decimal4(-1, 9), "-0.000000001"),
+ (timestamp(1_500_000, ntz=True), '"1970-01-01 00:00:01.5"'),
+ (timestamp(1, ntz=False), '"1970-01-01 00:00:00.000001+00:00"'),
+ ]:
+ with self.subTest(expected=expected):
+ self.assertEqual(v.toJson(), expected)
+ # The output must always be valid JSON.
+ json.loads(v.toJson())
+ # The output must not depend on the caller's decimal context.
+ with decimal.localcontext() as ctx:
+ ctx.prec = 5
+ self.assertEqual(v.toJson(), expected)
+
+ self.assertEqual(
+ timestamp(0, ntz=False).toJson("Asia/Kolkata"), '"1970-01-01
05:30:00+05:30"'
Review Comment:
Added both, `0.1 + 0.2` and the 1800-01-01 Los Angeles LMT case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
