gene-db commented on code in PR #45826: URL: https://github.com/apache/spark/pull/45826#discussion_r1556457254
########## python/pyspark/sql/tests/test_types.py: ########## @@ -1406,6 +1407,68 @@ def test_calendar_interval_type_with_sf(self): schema1 = self.spark.range(1).select(F.make_interval(F.lit(1))).schema self.assertEqual(schema1.fields[0].dataType, CalendarIntervalType()) + def test_variant_type(self): + from decimal import Decimal + self.assertEqual(VariantType().simpleString(), "variant") + + # Holds a tuple of (key, json string value, python value) + expected_values = [ + ("str", '"%s"' % ("0123456789" * 10), "0123456789" * 10), + ("short_str", '"abc"', "abc"), + ("null", "null", None), + ("true", "true", True), + ("false", "false", False), + ("int1", "1", 1), + ("-int1", "-5", -5), + ("int2", "257", 257), + ("-int2", "-124", -124), + ("int4", "65793", 65793), + ("-int4", "-69633", -69633), + ("int8", "4295033089", 4295033089), + ("-int8", "-4294967297", -4294967297), + ("float4", "1.23456789e-30", 1.23456789e-30), + ("-float4", "-4.56789e+29", -4.56789e+29), + ("dec4", "123.456", Decimal("123.456")), + ("-dec4", "-321.654", Decimal("-321.654")), + ("dec8", "429.4967297", Decimal("429.4967297")), + ("-dec8", "-5.678373902", Decimal("-5.678373902")), + ("dec16", "467440737095.51617", Decimal("467440737095.51617")), + ("-dec16", "-67.849438003827263", Decimal("-67.849438003827263")), + ("arr", '[1.1,"2",[3],{"4":5}]', [Decimal("1.1"), "2", [3], {"4": 5}]), + ("obj", '{"a":["123",{"b":2}],"c":3}', {"a": ["123", {"b": 2}], "c": 3}), + ] + json_str = "{%s}" % ",".join(['"%s": %s' % (t[0], t[1]) for t in expected_values]) + + df = self.spark.createDataFrame([({"json": json_str})]) Review Comment: @zhengruifeng Would it be ok to handle that implementation in a separate PR? This one is already getting pretty large. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org