ajantha-bhat commented on a change in pull request #3664: [CARBONDATA-3740] Add
line separator option to load command to configure the line separator during
csv parsing.
URL: https://github.com/apache/carbondata/pull/3664#discussion_r400682260
##########
File path:
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala
##########
@@ -77,4 +87,54 @@ class TestLoadOptions extends QueryTest with
BeforeAndAfterAll{
Row(1, "2015/7/23", "ind", "aaa1", "phone197", "ASD69643a", 15000))
}
+ test("test load data with line separator option value as Linux/Unix
\"\\n\"") {
+ sql(s"""LOAD DATA LOCAL INPATH
'$resourcesPath/carriage_return_in_string.csv' INTO TABLE
+ |carriage_return_in_string OPTIONS ('fileheader'='id, name, city',
'line_separator'='\\n')"""
+ .stripMargin.replace('\n', ' '));
+ checkAnswer(sql("select * from carriage_return_in_string where id = 1"),
+ Row(1, "2\r", "3"))
+ }
+
+ test("test load data without line separator option") {
+ sql(s"""LOAD DATA LOCAL INPATH
'$resourcesPath/carriage_return_in_string.csv' INTO TABLE
+ |carriage_return_in_string OPTIONS ('fileheader'='id, name,
city')"""
+ .stripMargin.replace('\n', ' '));
+ checkAnswer(sql("select * from carriage_return_in_string where id = 1"),
+ Row(1, "2", null))
+ }
+
+ test("test load data with line separator option value as Windows
\"\\r\\n\"") {
+ sql(s"""LOAD DATA LOCAL INPATH
'$resourcesPath/carriage_return_in_string.csv' INTO TABLE
+ |carriage_return_in_string OPTIONS ('fileheader'='id, name, city',
+ |'line_separator'='\\r\\n')""".stripMargin.replace('\n', ' '));
+ checkAnswer(sql("select * from carriage_return_in_string where id = 1"),
+ Row(1, "2\r", "3\n4"))
+ }
+
+ test("test load data with line separator option value as any two characters
\"ab\"") {
+ sql(s"""LOAD DATA LOCAL INPATH
'$resourcesPath/carriage_return_in_string.csv' INTO TABLE
Review comment:
In this case it should throw an exception ? as it is invalid line separator?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
