cloud-fan commented on code in PR #45657:
URL: https://github.com/apache/spark/pull/45657#discussion_r1535133900
##########
sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala:
##########
@@ -603,4 +605,118 @@ class CsvFunctionsSuite extends QueryTest with
SharedSparkSession {
$"csv", schema_of_csv("1,2\n2"), Map.empty[String, String].asJava))
checkAnswer(actual, Row(Row(1, "2\n2")))
}
+
+ test("SPARK-47497: to_csv support the data of ArrayType as pretty strings") {
+ val rows = new java.util.ArrayList[Row]()
+ rows.add(Row(1L, Row(2L, "Alice", Array(100L, 200L, null, 300L))))
+
+ val valueSchema = StructType(Seq(
+ StructField("age", LongType),
+ StructField("name", StringType),
+ StructField("scores", ArrayType(LongType))))
+ val schema = StructType(Seq(
+ StructField("key", LongType),
+ StructField("value", valueSchema)))
+
+ val df = spark.createDataFrame(rows, schema)
+ val actual = df.select(to_csv($"value"))
+ checkAnswer(actual, Row("2,Alice,\"[100, 200, null, 300]\""))
+ }
+
+ test("SPARK-47497: to_csv support the data of MapType as pretty strings") {
+ val rows = new java.util.ArrayList[Row]()
+ rows.add(Row(1L, Row(2L, "Alice",
+ Map("math" -> 100L, "english" -> 200L, "science" -> null))))
+
+ val valueSchema = StructType(Seq(
+ StructField("age", LongType),
+ StructField("name", StringType),
+ StructField("scores", MapType(StringType, LongType))))
+ val schema = StructType(Seq(
+ StructField("key", LongType),
+ StructField("value", valueSchema)))
+
+ val df = spark.createDataFrame(rows, schema)
+ val actual = df.select(to_csv($"value"))
+ checkAnswer(actual, Row("2,Alice,\"{math -> 100, english -> 200, science
-> null}\""))
+ }
+
+ test("SPARK-47497: to_csv support the data of StructType as pretty strings")
{
+ val rows = new java.util.ArrayList[Row]()
+ rows.add(Row(1L, Row(2L, "Alice", Row(100L, 200L, null))))
+
+ val valueSchema = StructType(Seq(
+ StructField("age", LongType),
+ StructField("name", StringType),
+ StructField("scores", StructType(Seq(
+ StructField("id1", LongType),
+ StructField("id2", LongType),
+ StructField("id3", LongType))))))
+ val schema = StructType(Seq(
+ StructField("key", LongType),
+ StructField("value", valueSchema)))
+
+ val df = spark.createDataFrame(rows, schema)
+ val actual = df.select(to_csv($"value"))
+ checkAnswer(actual, Row("2,Alice,\"{100, 200, null}\""))
+ }
+
+ test("SPARK-47497: to_csv support the data of BinaryType as pretty strings")
{
+ val rows = new java.util.ArrayList[Row]()
+ rows.add(Row(1L, Row(2L, "Alice", "a".getBytes(StandardCharsets.UTF_8))))
+
+ val valueSchema = StructType(Seq(
+ StructField("age", LongType),
+ StructField("name", StringType),
+ StructField("b", BinaryType)))
+ val schema = StructType(Seq(
+ StructField("key", LongType),
+ StructField("value", valueSchema)))
+
+ val df = spark.createDataFrame(rows, schema)
+ val actual = df.select(to_csv($"value"))
+ checkAnswer(actual, Row("2,Alice,[61]"))
+ }
+
+ test("SPARK-47497: to_csv can display VariantType data") {
+ val rows = new java.util.ArrayList[Row]()
+ rows.add(Row(1L, Row(2L, "Alice", new VariantVal(Array[Byte](1, 2, 3),
Array[Byte](4, 5)))))
+
+ val valueSchema = StructType(Seq(
+ StructField("age", LongType),
+ StructField("name", StringType),
+ StructField("v", VariantType)))
+ val schema = StructType(Seq(
+ StructField("key", LongType),
+ StructField("value", valueSchema)))
+
+ val df = spark.createDataFrame(rows, schema)
+ val actual = df.select(to_csv($"value"))
+ checkAnswer(actual, Row("2,Alice,\"\""))
Review Comment:
I'm fine to ban variant type in `to_csv`, as it's new.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
