Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/22379#discussion_r223039662
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
---
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import java.util.Calendar
+
+import scala.util.Random
+
+import org.scalatest.exceptions.TestFailedException
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.plans.PlanTestBase
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
+
+class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
with PlanTestBase {
+ val badCsv = "\u0000\u0000\u0000A\u0001AAA"
+
+ val gmtId = Option(DateTimeUtils.TimeZoneGMT.getID)
+
+ test("from_csv") {
+ val csvData = "1"
+ val schema = StructType(StructField("a", IntegerType) :: Nil)
+ checkEvaluation(
+ CsvToStructs(schema, Map.empty, Literal(csvData), gmtId),
+ InternalRow(1)
+ )
+ }
+
+ test("from_csv - invalid data") {
+ val csvData = "---"
+ val schema = StructType(StructField("a", DoubleType) :: Nil)
+ checkEvaluation(
+ CsvToStructs(schema, Map("mode" -> PermissiveMode.name),
Literal(csvData), gmtId),
+ InternalRow(null))
+
+ // Default mode is Permissive
+ checkEvaluation(CsvToStructs(schema, Map.empty, Literal(csvData),
gmtId), InternalRow(null))
+ }
+
+ test("from_csv null input column") {
+ val schema = StructType(StructField("a", IntegerType) :: Nil)
+ checkEvaluation(
+ CsvToStructs(schema, Map.empty, Literal.create(null, StringType),
gmtId),
+ null
+ )
+ }
+
+ test("from_csv bad UTF-8") {
+ val schema = StructType(StructField("a", IntegerType) :: Nil)
+ checkEvaluation(
+ CsvToStructs(schema, Map.empty, Literal(badCsv), gmtId),
+ InternalRow(null))
+ }
+
+ test("from_csv with timestamp") {
+ val schema = StructType(StructField("t", TimestampType) :: Nil)
+
+ val csvData1 = "2016-01-01T00:00:00.123Z"
+ var c = Calendar.getInstance(DateTimeUtils.TimeZoneGMT)
+ c.set(2016, 0, 1, 0, 0, 0)
+ c.set(Calendar.MILLISECOND, 123)
+ checkEvaluation(
+ CsvToStructs(schema, Map.empty, Literal(csvData1), gmtId),
+ InternalRow(c.getTimeInMillis * 1000L)
+ )
+ // The result doesn't change because the CSV string includes timezone
string ("Z" here),
+ // which means the string represents the timestamp string in the
timezone regardless of
+ // the timeZoneId parameter.
+ checkEvaluation(
+ CsvToStructs(schema, Map.empty, Literal(csvData1), Option("PST")),
+ InternalRow(c.getTimeInMillis * 1000L)
+ )
+
+ val csvData2 = "2016-01-01T00:00:00"
+ for (tz <- Random.shuffle(DateTimeTestUtils.ALL_TIMEZONES).take(50)) {
--- End diff --
how long does this test take?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
