nealrichardson commented on a change in pull request #9878: URL: https://github.com/apache/arrow/pull/9878#discussion_r607322619
########## File path: r/tests/testthat/test-dplyr-string-functions.R ########## @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +library(dplyr) +library(stringr) + +test_that("sub and gsub", { + df <- tibble(x = c("Foo", "bar")) + + for(fixed in c(TRUE, FALSE)) { + + expect_dplyr_equal( + input %>% + transmute(x = sub("Foo", "baz", x, fixed = fixed)) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = sub("^B.+", "baz", x, ignore.case = FALSE, fixed = fixed)) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = sub("Foo", "baz", x, ignore.case = FALSE, fixed = fixed)) %>% + collect(), + df + ) + + # the tests below all use ignore.case = TRUE + # but base::sub and base::gsub ignore ignore.case = TRUE with a warning when fixed = TRUE + # so we can't use expect_dplyr_equal() for the tests below + expect_equal( + df %>% + Table$create() %>% + transmute(x = sub("O", "u", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + tibble(x = c("Fuo", "bar")) + ) + expect_equal( + df %>% + Table$create() %>% + transmute(x = gsub("o", "u", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + tibble(x = c("Fuu", "bar")) + ) + expect_equal( + df %>% + Table$create() %>% + transmute(x = sub("^B.+", "baz", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + if (fixed) tibble(x = c("Foo", "bar")) else tibble(x = c("Foo", "baz")) + ) + + } + +}) + +test_that("str_replace and str_replace_all", { + df <- tibble(x = c("Foo", "bar")) + + library(stringr) + + expect_dplyr_equal( + input %>% + transmute(x = str_replace_all(x, regex("^F"), "baz")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, regex("^f[A-Z]{2}", ignore_case = TRUE), "baz")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace_all(x, fixed("o"), "u")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, fixed("O"), "u")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, fixed("O", ignore_case = TRUE), "u")) %>% + collect(), + df + ) + +}) + +test_that("backreferences", { + df <- tibble(x = c("Foo", "bar")) + + expect_dplyr_equal( + input %>% + transmute(desc = sub( + "(?:https?|ftp)://([^/\r\n]+)(/[^\r\n]*)?", + "path `\\2` on server `\\1`", + url + ) + ) %>% + collect(), + tibble(url = "https://arrow.apache.org/docs/r/";) + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, regex("^(\\w)o(.*)", ignore_case = TRUE), "\\1\\2p")) %>% + collect(), + df + ) +}) + +test_that("edge cases", { Review comment: Why are these not `expect_dplyr_equal`? ########## File path: r/tests/testthat/test-dplyr-string-functions.R ########## @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +library(dplyr) +library(stringr) + +test_that("sub and gsub", { + df <- tibble(x = c("Foo", "bar")) + + for(fixed in c(TRUE, FALSE)) { + + expect_dplyr_equal( + input %>% + transmute(x = sub("Foo", "baz", x, fixed = fixed)) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = sub("^B.+", "baz", x, ignore.case = FALSE, fixed = fixed)) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = sub("Foo", "baz", x, ignore.case = FALSE, fixed = fixed)) %>% + collect(), + df + ) + + # the tests below all use ignore.case = TRUE + # but base::sub and base::gsub ignore ignore.case = TRUE with a warning when fixed = TRUE + # so we can't use expect_dplyr_equal() for the tests below + expect_equal( + df %>% + Table$create() %>% + transmute(x = sub("O", "u", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + tibble(x = c("Fuo", "bar")) + ) + expect_equal( + df %>% + Table$create() %>% + transmute(x = gsub("o", "u", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + tibble(x = c("Fuu", "bar")) + ) + expect_equal( + df %>% + Table$create() %>% + transmute(x = sub("^B.+", "baz", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + if (fixed) tibble(x = c("Foo", "bar")) else tibble(x = c("Foo", "baz")) + ) + + } + +}) + +test_that("str_replace and str_replace_all", { + df <- tibble(x = c("Foo", "bar")) + + library(stringr) + + expect_dplyr_equal( + input %>% + transmute(x = str_replace_all(x, regex("^F"), "baz")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, regex("^f[A-Z]{2}", ignore_case = TRUE), "baz")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace_all(x, fixed("o"), "u")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, fixed("O"), "u")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, fixed("O", ignore_case = TRUE), "u")) %>% + collect(), + df + ) + +}) + +test_that("backreferences", { + df <- tibble(x = c("Foo", "bar")) + + expect_dplyr_equal( + input %>% + transmute(desc = sub( + "(?:https?|ftp)://([^/\r\n]+)(/[^\r\n]*)?", + "path `\\2` on server `\\1`", + url + ) + ) %>% + collect(), + tibble(url = "https://arrow.apache.org/docs/r/";) + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, regex("^(\\w)o(.*)", ignore_case = TRUE), "\\1\\2p")) %>% + collect(), + df + ) +}) + +test_that("edge cases", { + + # in case-insensitive fixed replace, test that "\\E" in the search string and + # backslashes in the replacement string are interpreted literally + expect_equal( + tibble(x = c("\\Q\\e\\D")) %>% + Table$create() %>% + transmute(x = sub("\\E", "\\L", x, ignore.case = TRUE, fixed = TRUE)) %>% + collect(), + tibble(x = c("\\Q\\L\\D")) + ) + + # test that a user's "(?i)" prefix does not break the "(?i)" prefix that's + # added in case-insensitive regex replace + expect_equal( + tibble(x = c("ABC")) %>% + Table$create() %>% + transmute(x = sub("(?i)^[abc]{3}$", "123", x, ignore.case = TRUE, fixed = FALSE)) %>% + collect(), + tibble(x = c("123")) + ) + +}) + +test_that("errors and warnings", { + skip_if_not_available("dataset") + + df <- tibble(x = c("Foo", "bar")) + ds <- InMemoryDataset$create(df) Review comment: Do these tests require a dataset? If so, can you comment why? If not, it's confusing that you're switching to a dataset here. ########## File path: r/tests/testthat/test-dplyr-string-functions.R ########## @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +library(dplyr) +library(stringr) + +test_that("sub and gsub", { + df <- tibble(x = c("Foo", "bar")) + + for(fixed in c(TRUE, FALSE)) { + + expect_dplyr_equal( + input %>% + transmute(x = sub("Foo", "baz", x, fixed = fixed)) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = sub("^B.+", "baz", x, ignore.case = FALSE, fixed = fixed)) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = sub("Foo", "baz", x, ignore.case = FALSE, fixed = fixed)) %>% + collect(), + df + ) + + # the tests below all use ignore.case = TRUE + # but base::sub and base::gsub ignore ignore.case = TRUE with a warning when fixed = TRUE + # so we can't use expect_dplyr_equal() for the tests below + expect_equal( + df %>% + Table$create() %>% + transmute(x = sub("O", "u", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + tibble(x = c("Fuo", "bar")) + ) + expect_equal( + df %>% + Table$create() %>% + transmute(x = gsub("o", "u", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + tibble(x = c("Fuu", "bar")) + ) + expect_equal( + df %>% + Table$create() %>% + transmute(x = sub("^B.+", "baz", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + if (fixed) tibble(x = c("Foo", "bar")) else tibble(x = c("Foo", "baz")) + ) + + } + +}) + +test_that("str_replace and str_replace_all", { + df <- tibble(x = c("Foo", "bar")) + + library(stringr) + + expect_dplyr_equal( + input %>% + transmute(x = str_replace_all(x, regex("^F"), "baz")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, regex("^f[A-Z]{2}", ignore_case = TRUE), "baz")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace_all(x, fixed("o"), "u")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, fixed("O"), "u")) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, fixed("O", ignore_case = TRUE), "u")) %>% + collect(), + df + ) + +}) + +test_that("backreferences", { + df <- tibble(x = c("Foo", "bar")) + + expect_dplyr_equal( + input %>% + transmute(desc = sub( + "(?:https?|ftp)://([^/\r\n]+)(/[^\r\n]*)?", + "path `\\2` on server `\\1`", + url + ) + ) %>% + collect(), + tibble(url = "https://arrow.apache.org/docs/r/";) + ) + expect_dplyr_equal( + input %>% + transmute(x = str_replace(x, regex("^(\\w)o(.*)", ignore_case = TRUE), "\\1\\2p")) %>% + collect(), + df + ) +}) + +test_that("edge cases", { + + # in case-insensitive fixed replace, test that "\\E" in the search string and + # backslashes in the replacement string are interpreted literally + expect_equal( + tibble(x = c("\\Q\\e\\D")) %>% + Table$create() %>% + transmute(x = sub("\\E", "\\L", x, ignore.case = TRUE, fixed = TRUE)) %>% + collect(), + tibble(x = c("\\Q\\L\\D")) + ) + + # test that a user's "(?i)" prefix does not break the "(?i)" prefix that's + # added in case-insensitive regex replace + expect_equal( + tibble(x = c("ABC")) %>% + Table$create() %>% + transmute(x = sub("(?i)^[abc]{3}$", "123", x, ignore.case = TRUE, fixed = FALSE)) %>% + collect(), + tibble(x = c("123")) + ) + +}) + +test_that("errors and warnings", { + skip_if_not_available("dataset") + + df <- tibble(x = c("Foo", "bar")) + ds <- InMemoryDataset$create(df) + + expect_error( + ds %>% transmute(x = str_replace_all(x, coll("o", locale = "en"), "รณ")), + "not supported" Review comment: Can you assert the complete error messages for these? I see some possibly complex logic in assembling the messages, and I personally find it all too easy to mess up the whitespace in those. ########## File path: r/tests/testthat/test-dplyr-string-functions.R ########## @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +library(dplyr) +library(stringr) + +test_that("sub and gsub", { + df <- tibble(x = c("Foo", "bar")) + + for(fixed in c(TRUE, FALSE)) { + + expect_dplyr_equal( + input %>% + transmute(x = sub("Foo", "baz", x, fixed = fixed)) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = sub("^B.+", "baz", x, ignore.case = FALSE, fixed = fixed)) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = sub("Foo", "baz", x, ignore.case = FALSE, fixed = fixed)) %>% + collect(), + df + ) + + # the tests below all use ignore.case = TRUE + # but base::sub and base::gsub ignore ignore.case = TRUE with a warning when fixed = TRUE + # so we can't use expect_dplyr_equal() for the tests below Review comment: I would split these out of your for loop, include the ones you can as `expect_dplyr_equal`, and make explicit the exceptions. ########## File path: r/tests/testthat/test-dplyr-string-functions.R ########## @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +library(dplyr) +library(stringr) + +test_that("sub and gsub", { + df <- tibble(x = c("Foo", "bar")) + + for(fixed in c(TRUE, FALSE)) { + + expect_dplyr_equal( + input %>% + transmute(x = sub("Foo", "baz", x, fixed = fixed)) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = sub("^B.+", "baz", x, ignore.case = FALSE, fixed = fixed)) %>% + collect(), + df + ) + expect_dplyr_equal( + input %>% + transmute(x = sub("Foo", "baz", x, ignore.case = FALSE, fixed = fixed)) %>% + collect(), + df + ) + + # the tests below all use ignore.case = TRUE + # but base::sub and base::gsub ignore ignore.case = TRUE with a warning when fixed = TRUE + # so we can't use expect_dplyr_equal() for the tests below + expect_equal( + df %>% + Table$create() %>% + transmute(x = sub("O", "u", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + tibble(x = c("Fuo", "bar")) + ) + expect_equal( + df %>% + Table$create() %>% + transmute(x = gsub("o", "u", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + tibble(x = c("Fuu", "bar")) + ) + expect_equal( + df %>% + Table$create() %>% + transmute(x = sub("^B.+", "baz", x, ignore.case = TRUE, fixed = fixed)) %>% + collect(), + if (fixed) tibble(x = c("Foo", "bar")) else tibble(x = c("Foo", "baz")) Review comment: Another reason to split this out of the for loop ########## File path: r/R/dplyr.R ########## @@ -434,6 +438,75 @@ build_function_list <- function(FUN) { ) } +arrow_r_string_replace_function <- function(FUN, max_replacements) { + function(pattern, replacement, x, ignore.case = FALSE, fixed = FALSE) { + if (ignore.case) { + if (fixed) { + pattern <- paste0("(?i)\\Q", pattern, "\\E") + } else { + pattern <- paste0("(?i)", pattern) + } + } + FUN( + ifelse(fixed && !ignore.case, "replace_substring", "replace_substring_regex"), + x, + options = list( + pattern = pattern, + replacement = replacement, + max_replacements = max_replacements + ) + ) + } +} + +arrow_stringr_string_replace_function <- function(FUN, max_replacements) { + function(string, pattern, replacement) { + # Assign stringr pattern modifier functions locally Review comment: Ok, fine by me ########## File path: r/R/dplyr.R ########## @@ -434,6 +438,77 @@ build_function_list <- function(FUN) { ) } +arrow_r_string_replace_function <- function(FUN, max_replacements) { + function(pattern, replacement, x, ignore.case = FALSE, fixed = FALSE) { + if (ignore.case) { + if (fixed) { + pattern <- gsub("\\E", "\\e", pattern, fixed = TRUE) Review comment: Could you leave some comments and/or links to the relevant regexp/re2 docs that explain why you're making these modifications? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
