[
https://issues.apache.org/jira/browse/ARROW-14908?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17475807#comment-17475807
]
Jeanette Clark edited comment on ARROW-14908 at 1/13/22, 10:28 PM:
-------------------------------------------------------------------
I am seeing similar issues (but without the summarize). The crash only occurs
after the join, filtering, and even collecting the entire (large) table works
fine. Example I was working off of and session info are below.
```library(arrow)
library(tidyverse)
library(janitor)
dir_out <- tempdir()
iris_clean <- clean_names(iris)
write_dataset(iris_clean,
file.path(dir_out, "iris"))
species_codes <- tribble(~species, ~code,
"setosa", "SET",
"versicolor", "VER",
"virginica", "VIR")
iris <- open_dataset(file.path(dir_out, "iris"))
d_j <- left_join(iris, species_codes) %>%
collect()```
```R version 4.1.2 (2021-11-01)
Platform: x86_64-w64-mingw32/x64 (64-bit)
Running under: Windows 10 x64 (build 19044)
Matrix products: default
locale:
[1] LC_COLLATE=English_United States.1252 LC_CTYPE=English_United States.1252
LC_MONETARY=English_United States.1252 LC_NUMERIC=C
[5] LC_TIME=English_United States.1252
attached base packages:
[1] stats graphics grDevices utils datasets methods base
other attached packages:
[1] janitor_2.1.0 forcats_0.5.1 stringr_1.4.0 dplyr_1.0.7 purrr_0.3.4
readr_2.1.1 tidyr_1.1.4 tibble_3.1.6 ggplot2_3.3.5 tidyverse_1.3.1 arrow_6.0.1
loaded via a namespace (and not attached):
[1] tidyselect_1.1.1 xfun_0.29 haven_2.4.3 snakecase_0.11.0 colorspace_2.0-2
vctrs_0.3.8 generics_0.1.1 htmltools_0.5.2 yaml_2.2.1 utf8_1.2.2
[11] rlang_0.4.12 pillar_1.6.4 glue_1.6.0 withr_2.4.3 DBI_1.1.2 bit64_4.0.5
dbplyr_2.1.1 modelr_0.1.8 readxl_1.3.1 lifecycle_1.0.1
[21] munsell_0.5.0 gtable_0.3.0 cellranger_1.1.0 rvest_1.0.2 evaluate_0.14
knitr_1.37 tzdb_0.2.0 fastmap_1.1.0 fansi_1.0.0 broom_0.7.11
[31] Rcpp_1.0.7 scales_1.1.1 backports_1.4.1 jsonlite_1.7.2 fs_1.5.2 bit_4.0.4
hms_1.1.1 digest_0.6.29 stringi_1.7.6 grid_4.1.2
[41] cli_3.1.0 tools_4.1.2 magrittr_2.0.1 crayon_1.4.2 pkgconfig_2.0.3
ellipsis_0.3.2 xml2_1.3.3 reprex_2.0.1 lubridate_1.8.0 rstudioapi_0.13
[51] assertthat_0.2.1 rmarkdown_2.11 httr_1.4.2 R6_2.5.1 compiler_4.1.2```
was (Author: JIRAUSER281420):
I am seeing similar issues (but without the summarize). The crash only occurs
after the join, filtering, and even collecting the entire (large) table works
fine. Example I was working off of and session info are below.
```
library(arrow)
library(tidyverse)
library(janitor)
dir_out <- tempdir()
iris_clean <- clean_names(iris)
write_dataset(iris_clean,
file.path(dir_out, "iris"))
species_codes <- tribble(~species, ~code,
"setosa", "SET",
"versicolor", "VER",
"virginica", "VIR")
iris <- open_dataset(file.path(dir_out, "iris"))
d_j <- left_join(iris, species_codes) %>%
collect()
```
```
R version 4.1.2 (2021-11-01)
Platform: x86_64-w64-mingw32/x64 (64-bit)
Running under: Windows 10 x64 (build 19044)
Matrix products: default
locale:
[1] LC_COLLATE=English_United States.1252 LC_CTYPE=English_United States.1252
LC_MONETARY=English_United States.1252 LC_NUMERIC=C
[5] LC_TIME=English_United States.1252
attached base packages:
[1] stats graphics grDevices utils datasets methods base
other attached packages:
[1] janitor_2.1.0 forcats_0.5.1 stringr_1.4.0 dplyr_1.0.7 purrr_0.3.4
readr_2.1.1 tidyr_1.1.4 tibble_3.1.6 ggplot2_3.3.5 tidyverse_1.3.1 arrow_6.0.1
loaded via a namespace (and not attached):
[1] tidyselect_1.1.1 xfun_0.29 haven_2.4.3 snakecase_0.11.0 colorspace_2.0-2
vctrs_0.3.8 generics_0.1.1 htmltools_0.5.2 yaml_2.2.1 utf8_1.2.2
[11] rlang_0.4.12 pillar_1.6.4 glue_1.6.0 withr_2.4.3 DBI_1.1.2 bit64_4.0.5
dbplyr_2.1.1 modelr_0.1.8 readxl_1.3.1 lifecycle_1.0.1
[21] munsell_0.5.0 gtable_0.3.0 cellranger_1.1.0 rvest_1.0.2 evaluate_0.14
knitr_1.37 tzdb_0.2.0 fastmap_1.1.0 fansi_1.0.0 broom_0.7.11
[31] Rcpp_1.0.7 scales_1.1.1 backports_1.4.1 jsonlite_1.7.2 fs_1.5.2 bit_4.0.4
hms_1.1.1 digest_0.6.29 stringi_1.7.6 grid_4.1.2
[41] cli_3.1.0 tools_4.1.2 magrittr_2.0.1 crayon_1.4.2 pkgconfig_2.0.3
ellipsis_0.3.2 xml2_1.3.3 reprex_2.0.1 lubridate_1.8.0 rstudioapi_0.13
[51] assertthat_0.2.1 rmarkdown_2.11 httr_1.4.2 R6_2.5.1 compiler_4.1.2
```
> [R] join on dataset crashes on Windows
> --------------------------------------
>
> Key: ARROW-14908
> URL: https://issues.apache.org/jira/browse/ARROW-14908
> Project: Apache Arrow
> Issue Type: Bug
> Components: R
> Affects Versions: 6.0.0
> Environment: R version 4.0.4
> Reporter: Fabio Machado
> Priority: Critical
>
> {code:java}
> library(tidyverse)
> library(arrow)
> car_info <- rownames_to_column(mtcars, "car_info")
> cars_arrow_table <- arrow_table(car_info)
> other_mtcars_data <- select(car_info, 1) %>%
> mutate(main_color = sample( c("red", "blue", "white", "black"), size = n(),
> replace = TRUE)) %>%
> arrow::arrow_table()
> temp <- tempdir()
> par_temp <- paste0(temp, "\\parquet")
> car_info %>% arrow::write_dataset(par_temp)
> cars_arrow <- arrow::open_dataset(par_temp)
> # using arrow tables works
> ------------------------------------------------------
> cars_arrow_table %>% left_join(other_mtcars_data) %>% count(main_color) %>%
> collect()
> # using open dataset crashes R
> ------------------------------------------------------------------
> other_mtcars_data %>%
> left_join(cars_arrow) %>%
> count(main_color) %>%
> collect()
> #other variation also crash
> cars_arrow %>%
> left_join(other_mtcars_data) %>%
> count(main_color) %>%
> collect()
> cars_arrow %>%
> left_join(other_mtcars_data) %>%
> group_by(main_color) %>%
> summarise(n = n()) %>%
> collect()
> #compute also crashes
> cars_arrow %>%
> left_join(other_mtcars_data) %>%
> count(main_color) %>%
> compute()
> # workaround with duckdb
> ------------------------------------------------------
> ##this works
> cars_duck <- to_duckdb(cars_arrow, auto_disconnect = TRUE)
> other_cars_duck <- to_duckdb(other_mtcars_data, auto_disconnect = TRUE)
>
> cars_duck %>%
> left_join(other_cars_duck) %>%
> count(main_color) %>%
> collect()
> ##this doesn't (don't know if expected to work actually)
> cars_arrow %>%
> left_join(other_mtcars_data) %>%
> to_duckdb() {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
