thisisnic commented on code in PR #33614:
URL: https://github.com/apache/arrow/pull/33614#discussion_r1072273292
##########
r/tests/testthat/test-dataset-csv.R:
##########
@@ -476,3 +476,87 @@ test_that("CSV reading/parsing/convert options can be
passed in as lists", {
expect_equal(ds1, ds2)
})
+
+test_that("open_delim_dataset params passed through to open_dataset", {
+ ds <- open_delim_dataset(csv_dir, delim = ",", partitioning = "part")
+ expect_r6_class(ds$format, "CsvFileFormat")
+ expect_r6_class(ds$filesystem, "LocalFileSystem")
+ expect_identical(names(ds), c(names(df1), "part"))
+ expect_identical(dim(ds), c(20L, 7L))
+
+ # quote
+ df <- data.frame(a = c(1, 2), b = c("'abc'", "'def'"))
+ dst_dir <- make_temp_dir()
+ dst_file <- file.path(dst_dir, "data.csv")
+ write.table(df, sep = ",", dst_file, row.names = FALSE, quote = FALSE)
+
+ ds_quote <- open_csv_dataset(dst_dir, quote = "'") %>% collect()
+ expect_equal(ds_quote$b, c("abc", "def"))
+
+ # na
+ ds <- open_csv_dataset(csv_dir, partitioning = "part", na = c("", "NA",
"FALSE")) %>% collect()
+ expect_identical(ds$lgl, c(
+ TRUE, NA, NA, TRUE, NA, TRUE, NA, NA, TRUE, NA, TRUE, NA, NA,
+ TRUE, NA, TRUE, NA, NA, TRUE, NA
+ ))
+
+ # col_names and skip
+ ds <- open_csv_dataset(
+ csv_dir,
+ partitioning = "part",
+ col_names = paste0("col_", 1:6),
+ skip = 1
+ ) %>% collect()
+
+ expect_named(ds, c("col_1", "col_2", "col_3", "col_4", "col_5", "col_6",
"part"))
+ expect_equal(nrow(ds), 20)
+
+ # col_types
+ df <- data.frame(a = c(1, NA, 2), b = c("'abc'", NA, "'def'"))
+ dst_dir <- make_temp_dir()
+ dst_file <- file.path(dst_dir, "data.csv")
+ write.table(df, sep = ",", dst_file, row.names = FALSE, quote = FALSE)
+
+ data_schema <- schema(a = string(), b = string())
+ ds_strings <- open_csv_dataset(dst_dir, col_types = data_schema)
+ expect_equal(ds_strings$schema, schema(a = string(), b = string()))
+
+ # skip_empty_rows
+ tf <- tempfile()
+ writeLines('"x"\n"y"\nNA\nNA\n"NULL"\n\n\n', tf)
+
+ ds <- open_csv_dataset(tf, skip_empty_rows = FALSE) %>% collect()
+ expect_equal(nrow(ds), 7)
+
+ # timestamp_parsers
+ df <- data.frame(time = "2023-01-16 19:47:57")
+ dst_dir <- make_temp_dir()
+ dst_file <- file.path(dst_dir, "data.csv")
+ write.table(df, sep = ",", dst_file, row.names = FALSE, quote = FALSE)
+
+ ds <- open_csv_dataset(dst_dir, timestamp_parsers =
c(TimestampParser$create(format = "%d-%m-%y"))) %>% collect()
+
+ # GH-33708: timestamp_parsers don't appear to be working properly
+ expect_error(
+ expect_equal(ds$time, "16-01-2023")
+ )
Review Comment:
The expected error message is "`object` (`actual`) not equal to `expected`
(`expected`)." but I didn't think I should be testing that given it's the
message from testthat? Comment above this line refers to the issue ticket I
opened, but I can add something else to make it more explicit perhaps?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
