This is an automated email from the ASF dual-hosted git repository.
npr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 36b5a96 ARROW-9219: [R] coerce_timestamps in Parquet write options
does not work
36b5a96 is described below
commit 36b5a9638f1debeadf2bd8f85bcf7aa3d5c30ae0
Author: Neal Richardson <[email protected]>
AuthorDate: Fri Jun 26 08:22:56 2020 -0700
ARROW-9219: [R] coerce_timestamps in Parquet write options does not work
In addition to fixing the bug (a quick fix), I also spent some time
deleting unnecessary bindings for parquet writer builder methods. There's more
that can be done, which I think would shave more off of the built library size,
but I don't have time to do it right now.
Closes #7550 from nealrichardson/r-fix-parquet
Authored-by: Neal Richardson <[email protected]>
Signed-off-by: Neal Richardson <[email protected]>
---
r/R/arrowExports.R | 52 +-------
r/R/parquet.R | 178 ++++++++++++---------------
r/src/arrowExports.cpp | 221 ++--------------------------------
r/src/parquet.cpp | 156 +++++++++---------------
r/tests/testthat/helper-expectation.R | 6 +
r/tests/testthat/test-parquet.R | 22 ++++
6 files changed, 169 insertions(+), 466 deletions(-)
diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R
index 4b6a5aa..54fadcb 100644
--- a/r/R/arrowExports.R
+++ b/r/R/arrowExports.R
@@ -1052,40 +1052,8 @@ parquet___arrow___FileReader__num_rows <-
function(reader){
.Call(`_arrow_parquet___arrow___FileReader__num_rows` , reader)
}
-parquet___ArrowWriterProperties___Builder__create <- function(){
- .Call(`_arrow_parquet___ArrowWriterProperties___Builder__create` )
-}
-
-parquet___ArrowWriterProperties___Builder__store_schema <- function(builder){
-
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__store_schema`
, builder))
-}
-
-parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps
<- function(builder){
-
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps`
, builder))
-}
-
-parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps
<- function(builder){
-
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps`
, builder))
-}
-
-parquet___ArrowWriterProperties___Builder__coerce_timestamps <-
function(builder, unit){
-
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__coerce_timestamps`
, builder, unit))
-}
-
-parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps <-
function(builder){
-
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps`
, builder))
-}
-
-parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps <-
function(builder){
-
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps`
, builder))
-}
-
-parquet___ArrowWriterProperties___Builder__build <- function(builder){
- .Call(`_arrow_parquet___ArrowWriterProperties___Builder__build` , builder)
-}
-
-parquet___default_writer_properties <- function(){
- .Call(`_arrow_parquet___default_writer_properties` )
+parquet___ArrowWriterProperties___create <-
function(allow_truncated_timestamps, use_deprecated_int96_timestamps,
timestamp_unit){
+ .Call(`_arrow_parquet___ArrowWriterProperties___create` ,
allow_truncated_timestamps, use_deprecated_int96_timestamps, timestamp_unit)
}
parquet___WriterProperties___Builder__create <- function(){
@@ -1096,30 +1064,14 @@ parquet___WriterProperties___Builder__version <-
function(builder, version){
invisible(.Call(`_arrow_parquet___WriterProperties___Builder__version` ,
builder, version))
}
-parquet___ArrowWriterProperties___Builder__default_compression <-
function(builder, compression){
-
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__default_compression`
, builder, compression))
-}
-
parquet___ArrowWriterProperties___Builder__set_compressions <-
function(builder, paths, types){
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__set_compressions`
, builder, paths, types))
}
-parquet___ArrowWriterProperties___Builder__default_compression_level <-
function(builder, compression_level){
-
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__default_compression_level`
, builder, compression_level))
-}
-
parquet___ArrowWriterProperties___Builder__set_compression_levels <-
function(builder, paths, levels){
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__set_compression_levels`
, builder, paths, levels))
}
-parquet___ArrowWriterProperties___Builder__default_write_statistics <-
function(builder, write_statistics){
-
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__default_write_statistics`
, builder, write_statistics))
-}
-
-parquet___ArrowWriterProperties___Builder__default_use_dictionary <-
function(builder, use_dictionary){
-
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__default_use_dictionary`
, builder, use_dictionary))
-}
-
parquet___ArrowWriterProperties___Builder__set_use_dictionary <-
function(builder, paths, use_dictionary){
invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__set_use_dictionary`
, builder, paths, use_dictionary))
}
diff --git a/r/R/parquet.R b/r/R/parquet.R
index c0c2eaf..e504a5c 100644
--- a/r/R/parquet.R
+++ b/r/R/parquet.R
@@ -174,51 +174,25 @@ write_parquet <- function(x,
invisible(x_out)
}
-
-ParquetArrowWriterPropertiesBuilder <-
R6Class("ParquetArrowWriterPropertiesBuilder", inherit = ArrowObject,
- public = list(
- store_schema = function() {
- parquet___ArrowWriterProperties___Builder__store_schema(self)
- self
- },
- set_int96_support = function(use_deprecated_int96_timestamps = FALSE) {
- if (use_deprecated_int96_timestamps) {
-
parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps(self)
- } else {
-
parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps(self)
- }
- self
- },
- set_coerce_timestamps = function(coerce_timestamps = NULL) {
- if (!is.null(coerce_timestamps)) {
- unit <- make_valid_time_unit(coerce_timestamps,
- c("ms" = TimeUnit$MILLI, "us" = TimeUnit$MICRO)
- )
- parquet___ArrowWriterProperties___Builder__coerce_timestamps(unit)
- }
- self
- },
- set_allow_truncated_timestamps = function(allow_truncated_timestamps =
FALSE) {
- if (allow_truncated_timestamps) {
-
parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps(self)
- } else {
-
parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps(self)
- }
-
- self
- }
-
- )
-)
ParquetArrowWriterProperties <- R6Class("ParquetArrowWriterProperties",
inherit = ArrowObject)
-
-ParquetArrowWriterProperties$create <-
function(use_deprecated_int96_timestamps = FALSE, coerce_timestamps = NULL,
allow_truncated_timestamps = FALSE) {
- builder <- shared_ptr(ParquetArrowWriterPropertiesBuilder,
parquet___ArrowWriterProperties___Builder__create())
- builder$store_schema()
- builder$set_int96_support(use_deprecated_int96_timestamps)
- builder$set_coerce_timestamps(coerce_timestamps)
- builder$set_allow_truncated_timestamps(allow_truncated_timestamps)
- shared_ptr(ParquetArrowWriterProperties,
parquet___ArrowWriterProperties___Builder__build(builder))
+ParquetArrowWriterProperties$create <-
function(use_deprecated_int96_timestamps = FALSE,
+ coerce_timestamps = NULL,
+ allow_truncated_timestamps =
FALSE) {
+ if (is.null(coerce_timestamps)) {
+ timestamp_unit <- -1L # null sentinel value
+ } else {
+ timestamp_unit <- make_valid_time_unit(coerce_timestamps,
+ c("ms" = TimeUnit$MILLI, "us" = TimeUnit$MICRO)
+ )
+ }
+ shared_ptr(
+ ParquetArrowWriterProperties,
+ parquet___ArrowWriterProperties___create(
+ use_deprecated_int96_timestamps =
isTRUE(use_deprecated_int96_timestamps),
+ timestamp_unit = timestamp_unit,
+ allow_truncated_timestamps = isTRUE(allow_truncated_timestamps)
+ )
+ )
}
valid_parquet_version <- c(
@@ -285,86 +259,89 @@ ParquetWriterPropertiesBuilder <-
R6Class("ParquetWriterPropertiesBuilder", inhe
set_version = function(version) {
parquet___ArrowWriterProperties___Builder__version(self,
make_valid_version(version))
},
-
- set_compression = function(table, compression){
- private$.set(table, compression_from_name(compression), "compression",
is.integer,
- parquet___ArrowWriterProperties___Builder__default_compression,
+ set_compression = function(table, compression) {
+ compression <- compression_from_name(compression)
+ assert_that(is.integer(compression))
+ private$.set(table, compression,
parquet___ArrowWriterProperties___Builder__set_compressions
)
},
-
set_compression_level = function(table, compression_level){
- private$.set(table, compression_level, "compression_level",
is_integerish,
- parquet___ArrowWriterProperties___Builder__default_compression_level,
+ assert_that(is_integerish(compression_level))
+ private$.set(table, compression_level,
parquet___ArrowWriterProperties___Builder__set_compression_levels
)
},
-
set_dictionary = function(table, use_dictionary) {
- private$.set(table, use_dictionary, "use_dictionary", is.logical,
- parquet___ArrowWriterProperties___Builder__default_use_dictionary,
+ assert_that(is.logical(use_dictionary))
+ private$.set(table, use_dictionary,
parquet___ArrowWriterProperties___Builder__set_use_dictionary
)
},
-
set_write_statistics = function(table, write_statistics) {
- private$.set(table, write_statistics, "write_statistics", is.logical,
- parquet___ArrowWriterProperties___Builder__default_write_statistics,
+ assert_that(is.logical(write_statistics))
+ private$.set(table, write_statistics,
parquet___ArrowWriterProperties___Builder__set_write_statistics
)
},
-
set_data_page_size = function(data_page_size) {
parquet___ArrowWriterProperties___Builder__data_page_size(self,
data_page_size)
}
),
private = list(
- .set = function(table, value, name, is, default, multiple) {
- msg <- paste0("unsupported ", name, "= specification")
- assert_that(is(value), msg = msg)
+ .set = function(table, value, FUN) {
+ msg <- paste0("unsupported ", substitute(value), "= specification")
column_names <- names(table)
- if (is.null(given_names <- names(value))) {
- if (length(value) == 1L) {
- default(self, value)
- } else if (length(value) == length(column_names)) {
- multiple(self, column_names, value)
+ given_names <- names(value)
+ if (is.null(given_names)) {
+ if (length(value) %in% c(1L, length(column_names))) {
+ # If there's a single, unnamed value, FUN will set it globally
+ # If there are values for all columns, send them along with the names
+ FUN(self, column_names, value)
+ } else {
+ abort(msg)
}
- } else if(all(given_names %in% column_names)) {
- multiple(self, given_names, value)
+ } else if (all(given_names %in% column_names)) {
+ # Use the given names
+ FUN(self, given_names, value)
} else {
abort(msg)
}
}
)
-
)
-ParquetWriterProperties$create <- function(table, version = NULL, compression
= NULL, compression_level = NULL, use_dictionary = NULL, write_statistics =
NULL, data_page_size = NULL) {
- if (is.null(version) && is.null(compression) && is.null(compression_level)
&& is.null(use_dictionary) && is.null(write_statistics) &&
is.null(data_page_size)) {
- shared_ptr(ParquetWriterProperties, parquet___default_writer_properties())
- } else {
- builder <- shared_ptr(ParquetWriterPropertiesBuilder,
parquet___WriterProperties___Builder__create())
- if (!is.null(version)) {
- builder$set_version(version)
- }
- if (!is.null(compression)) {
- builder$set_compression(table, compression = compression)
- }
- if (!is.null(compression_level)) {
- builder$set_compression_level(table, compression_level =
compression_level)
- }
- if (!is.null(use_dictionary)) {
- builder$set_dictionary(table, use_dictionary)
- }
- if (!is.null(write_statistics)) {
- builder$set_write_statistics(table, write_statistics)
- }
- if (!is.null(data_page_size)) {
- builder$set_data_page_size(data_page_size)
- }
- shared_ptr(ParquetWriterProperties,
parquet___WriterProperties___Builder__build(builder))
+ParquetWriterProperties$create <- function(table,
+ version = NULL,
+ compression = NULL,
+ compression_level = NULL,
+ use_dictionary = NULL,
+ write_statistics = NULL,
+ data_page_size = NULL) {
+ builder <- shared_ptr(
+ ParquetWriterPropertiesBuilder,
+ parquet___WriterProperties___Builder__create()
+ )
+ if (!is.null(version)) {
+ builder$set_version(version)
+ }
+ if (!is.null(compression)) {
+ builder$set_compression(table, compression = compression)
+ }
+ if (!is.null(compression_level)) {
+ builder$set_compression_level(table, compression_level = compression_level)
}
+ if (!is.null(use_dictionary)) {
+ builder$set_dictionary(table, use_dictionary)
+ }
+ if (!is.null(write_statistics)) {
+ builder$set_write_statistics(table, write_statistics)
+ }
+ if (!is.null(data_page_size)) {
+ builder$set_data_page_size(data_page_size)
+ }
+ shared_ptr(ParquetWriterProperties,
parquet___WriterProperties___Builder__build(builder))
}
#' @title ParquetFileWriter class
@@ -391,18 +368,13 @@ ParquetFileWriter <- R6Class("ParquetFileWriter", inherit
= ArrowObject,
WriteTable = function(table, chunk_size) {
parquet___arrow___FileWriter__WriteTable(self, table, chunk_size)
},
- Close = function() {
- parquet___arrow___FileWriter__Close(self)
- }
+ Close = function() parquet___arrow___FileWriter__Close(self)
)
-
)
-ParquetFileWriter$create <- function(
- schema,
- sink,
- properties = ParquetWriterProperties$create(),
- arrow_properties = ParquetArrowWriterProperties$create()
-) {
+ParquetFileWriter$create <- function(schema,
+ sink,
+ properties =
ParquetWriterProperties$create(),
+ arrow_properties =
ParquetArrowWriterProperties$create()) {
shared_ptr(
ParquetFileWriter,
parquet___arrow___ParquetFileWriter__Open(schema, sink, properties,
arrow_properties)
diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp
index 4231a8c..3fa77c2 100644
--- a/r/src/arrowExports.cpp
+++ b/r/src/arrowExports.cpp
@@ -4090,141 +4090,18 @@ RcppExport SEXP
_arrow_parquet___arrow___FileReader__num_rows(SEXP reader_sexp){
// parquet.cpp
#if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>
parquet___ArrowWriterProperties___Builder__create();
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__create(){
+std::shared_ptr<parquet::ArrowWriterProperties>
parquet___ArrowWriterProperties___create(bool allow_truncated_timestamps, bool
use_deprecated_int96_timestamps, int timestamp_unit);
+RcppExport SEXP _arrow_parquet___ArrowWriterProperties___create(SEXP
allow_truncated_timestamps_sexp, SEXP use_deprecated_int96_timestamps_sexp,
SEXP timestamp_unit_sexp){
BEGIN_RCPP
- return Rcpp::wrap(parquet___ArrowWriterProperties___Builder__create());
+ Rcpp::traits::input_parameter<bool>::type
allow_truncated_timestamps(allow_truncated_timestamps_sexp);
+ Rcpp::traits::input_parameter<bool>::type
use_deprecated_int96_timestamps(use_deprecated_int96_timestamps_sexp);
+ Rcpp::traits::input_parameter<int>::type
timestamp_unit(timestamp_unit_sexp);
+ return
Rcpp::wrap(parquet___ArrowWriterProperties___create(allow_truncated_timestamps,
use_deprecated_int96_timestamps, timestamp_unit));
END_RCPP
}
#else
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__create(){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__create(). Please use
arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
-void parquet___ArrowWriterProperties___Builder__store_schema(const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder);
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__store_schema(SEXP
builder_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>&>::type
builder(builder_sexp);
- parquet___ArrowWriterProperties___Builder__store_schema(builder);
- return R_NilValue;
-END_RCPP
-}
-#else
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__store_schema(SEXP
builder_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__store_schema(). Please use
arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
-void
parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps(const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder);
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps(SEXP
builder_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>&>::type
builder(builder_sexp);
-
parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps(builder);
- return R_NilValue;
-END_RCPP
-}
-#else
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps(SEXP
builder_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps().
Please use arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
-void
parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps(const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder);
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps(SEXP
builder_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>&>::type
builder(builder_sexp);
-
parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps(builder);
- return R_NilValue;
-END_RCPP
-}
-#else
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps(SEXP
builder_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps().
Please use arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
-void parquet___ArrowWriterProperties___Builder__coerce_timestamps(const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder,
arrow::TimeUnit::type unit);
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__coerce_timestamps(SEXP
builder_sexp, SEXP unit_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>&>::type
builder(builder_sexp);
- Rcpp::traits::input_parameter<arrow::TimeUnit::type>::type
unit(unit_sexp);
- parquet___ArrowWriterProperties___Builder__coerce_timestamps(builder,
unit);
- return R_NilValue;
-END_RCPP
-}
-#else
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__coerce_timestamps(SEXP
builder_sexp, SEXP unit_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__coerce_timestamps(). Please use
arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
-void
parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps(const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder);
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps(SEXP
builder_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>&>::type
builder(builder_sexp);
-
parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps(builder);
- return R_NilValue;
-END_RCPP
-}
-#else
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps(SEXP
builder_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps(). Please
use arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
-void
parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps(const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder);
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps(SEXP
builder_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>&>::type
builder(builder_sexp);
-
parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps(builder);
- return R_NilValue;
-END_RCPP
-}
-#else
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps(SEXP
builder_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps().
Please use arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<parquet::ArrowWriterProperties>
parquet___ArrowWriterProperties___Builder__build(const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder);
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__build(SEXP
builder_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>&>::type
builder(builder_sexp);
- return
Rcpp::wrap(parquet___ArrowWriterProperties___Builder__build(builder));
-END_RCPP
-}
-#else
-RcppExport SEXP _arrow_parquet___ArrowWriterProperties___Builder__build(SEXP
builder_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__build(). Please use
arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
-std::shared_ptr<parquet::WriterProperties>
parquet___default_writer_properties();
-RcppExport SEXP _arrow_parquet___default_writer_properties(){
-BEGIN_RCPP
- return Rcpp::wrap(parquet___default_writer_properties());
-END_RCPP
-}
-#else
-RcppExport SEXP _arrow_parquet___default_writer_properties(){
- Rf_error("Cannot call parquet___default_writer_properties(). Please use
arrow::install_arrow() to install required runtime libraries. ");
+RcppExport SEXP _arrow_parquet___ArrowWriterProperties___create(SEXP
allow_truncated_timestamps_sexp, SEXP use_deprecated_int96_timestamps_sexp,
SEXP timestamp_unit_sexp){
+ Rf_error("Cannot call parquet___ArrowWriterProperties___create().
Please use arrow::install_arrow() to install required runtime libraries. ");
}
#endif
@@ -4261,23 +4138,6 @@ RcppExport SEXP
_arrow_parquet___WriterProperties___Builder__version(SEXP builde
// parquet.cpp
#if defined(ARROW_R_WITH_ARROW)
-void parquet___ArrowWriterProperties___Builder__default_compression(const
std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const
arrow::Compression::type& compression);
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__default_compression(SEXP
builder_sexp, SEXP compression_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
- Rcpp::traits::input_parameter<const arrow::Compression::type&>::type
compression(compression_sexp);
- parquet___ArrowWriterProperties___Builder__default_compression(builder,
compression);
- return R_NilValue;
-END_RCPP
-}
-#else
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__default_compression(SEXP
builder_sexp, SEXP compression_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__default_compression(). Please use
arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
void parquet___ArrowWriterProperties___Builder__set_compressions(const
std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const
std::vector<std::string>& paths, const Rcpp::IntegerVector& types);
RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__set_compressions(SEXP
builder_sexp, SEXP paths_sexp, SEXP types_sexp){
BEGIN_RCPP
@@ -4296,23 +4156,6 @@ RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__set_compressio
// parquet.cpp
#if defined(ARROW_R_WITH_ARROW)
-void
parquet___ArrowWriterProperties___Builder__default_compression_level(const
std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, int
compression_level);
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__default_compression_level(SEXP
builder_sexp, SEXP compression_level_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
- Rcpp::traits::input_parameter<int>::type
compression_level(compression_level_sexp);
-
parquet___ArrowWriterProperties___Builder__default_compression_level(builder,
compression_level);
- return R_NilValue;
-END_RCPP
-}
-#else
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__default_compression_level(SEXP
builder_sexp, SEXP compression_level_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__default_compression_level(). Please
use arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
void parquet___ArrowWriterProperties___Builder__set_compression_levels(const
std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const
std::vector<std::string>& paths, const Rcpp::IntegerVector& levels);
RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__set_compression_levels(SEXP
builder_sexp, SEXP paths_sexp, SEXP levels_sexp){
BEGIN_RCPP
@@ -4331,40 +4174,6 @@ RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__set_compressio
// parquet.cpp
#if defined(ARROW_R_WITH_ARROW)
-void parquet___ArrowWriterProperties___Builder__default_write_statistics(const
std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, bool
write_statistics);
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__default_write_statistics(SEXP
builder_sexp, SEXP write_statistics_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
- Rcpp::traits::input_parameter<bool>::type
write_statistics(write_statistics_sexp);
-
parquet___ArrowWriterProperties___Builder__default_write_statistics(builder,
write_statistics);
- return R_NilValue;
-END_RCPP
-}
-#else
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__default_write_statistics(SEXP
builder_sexp, SEXP write_statistics_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__default_write_statistics(). Please
use arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
-void parquet___ArrowWriterProperties___Builder__default_use_dictionary(const
std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, bool
use_dictionary);
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__default_use_dictionary(SEXP
builder_sexp, SEXP use_dictionary_sexp){
-BEGIN_RCPP
- Rcpp::traits::input_parameter<const
std::shared_ptr<parquet::WriterPropertiesBuilder>&>::type builder(builder_sexp);
- Rcpp::traits::input_parameter<bool>::type
use_dictionary(use_dictionary_sexp);
-
parquet___ArrowWriterProperties___Builder__default_use_dictionary(builder,
use_dictionary);
- return R_NilValue;
-END_RCPP
-}
-#else
-RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__default_use_dictionary(SEXP
builder_sexp, SEXP use_dictionary_sexp){
- Rf_error("Cannot call
parquet___ArrowWriterProperties___Builder__default_use_dictionary(). Please use
arrow::install_arrow() to install required runtime libraries. ");
-}
-#endif
-
-// parquet.cpp
-#if defined(ARROW_R_WITH_ARROW)
void parquet___ArrowWriterProperties___Builder__set_use_dictionary(const
std::shared_ptr<parquet::WriterPropertiesBuilder>& builder, const
std::vector<std::string>& paths, const Rcpp::LogicalVector& use_dictionary);
RcppExport SEXP
_arrow_parquet___ArrowWriterProperties___Builder__set_use_dictionary(SEXP
builder_sexp, SEXP paths_sexp, SEXP use_dictionary_sexp){
BEGIN_RCPP
@@ -6119,23 +5928,11 @@ static const R_CallMethodDef CallEntries[] = {
{ "_arrow_parquet___arrow___FileReader__ReadTable1", (DL_FUNC)
&_arrow_parquet___arrow___FileReader__ReadTable1, 1},
{ "_arrow_parquet___arrow___FileReader__ReadTable2", (DL_FUNC)
&_arrow_parquet___arrow___FileReader__ReadTable2, 2},
{ "_arrow_parquet___arrow___FileReader__num_rows", (DL_FUNC)
&_arrow_parquet___arrow___FileReader__num_rows, 1},
- { "_arrow_parquet___ArrowWriterProperties___Builder__create",
(DL_FUNC) &_arrow_parquet___ArrowWriterProperties___Builder__create, 0},
- {
"_arrow_parquet___ArrowWriterProperties___Builder__store_schema", (DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__store_schema, 1},
- {
"_arrow_parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps,
1},
- {
"_arrow_parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps,
1},
- {
"_arrow_parquet___ArrowWriterProperties___Builder__coerce_timestamps",
(DL_FUNC) &_arrow_parquet___ArrowWriterProperties___Builder__coerce_timestamps,
2},
- {
"_arrow_parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps,
1},
- {
"_arrow_parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps,
1},
- { "_arrow_parquet___ArrowWriterProperties___Builder__build",
(DL_FUNC) &_arrow_parquet___ArrowWriterProperties___Builder__build, 1},
- { "_arrow_parquet___default_writer_properties", (DL_FUNC)
&_arrow_parquet___default_writer_properties, 0},
+ { "_arrow_parquet___ArrowWriterProperties___create", (DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___create, 3},
{ "_arrow_parquet___WriterProperties___Builder__create",
(DL_FUNC) &_arrow_parquet___WriterProperties___Builder__create, 0},
{ "_arrow_parquet___WriterProperties___Builder__version",
(DL_FUNC) &_arrow_parquet___WriterProperties___Builder__version, 2},
- {
"_arrow_parquet___ArrowWriterProperties___Builder__default_compression",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__default_compression, 2},
{
"_arrow_parquet___ArrowWriterProperties___Builder__set_compressions", (DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__set_compressions, 3},
- {
"_arrow_parquet___ArrowWriterProperties___Builder__default_compression_level",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__default_compression_level,
2},
{
"_arrow_parquet___ArrowWriterProperties___Builder__set_compression_levels",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__set_compression_levels, 3},
- {
"_arrow_parquet___ArrowWriterProperties___Builder__default_write_statistics",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__default_write_statistics,
2},
- {
"_arrow_parquet___ArrowWriterProperties___Builder__default_use_dictionary",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__default_use_dictionary, 2},
{
"_arrow_parquet___ArrowWriterProperties___Builder__set_use_dictionary",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__set_use_dictionary, 3},
{
"_arrow_parquet___ArrowWriterProperties___Builder__set_write_statistics",
(DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__set_write_statistics, 3},
{
"_arrow_parquet___ArrowWriterProperties___Builder__data_page_size", (DL_FUNC)
&_arrow_parquet___ArrowWriterProperties___Builder__data_page_size, 2},
diff --git a/r/src/parquet.cpp b/r/src/parquet.cpp
index 13d2d9a..626e58c 100644
--- a/r/src/parquet.cpp
+++ b/r/src/parquet.cpp
@@ -106,61 +106,27 @@ class ArrowWriterPropertiesBuilder : public
ArrowWriterProperties::Builder {
} // namespace parquet
// [[arrow::export]]
-std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>
-parquet___ArrowWriterProperties___Builder__create() {
- return std::make_shared<parquet::ArrowWriterPropertiesBuilder>();
-}
-
-// [[arrow::export]]
-void parquet___ArrowWriterProperties___Builder__store_schema(
- const std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder) {
+std::shared_ptr<parquet::ArrowWriterProperties>
parquet___ArrowWriterProperties___create(
+ bool allow_truncated_timestamps, bool use_deprecated_int96_timestamps,
+ int timestamp_unit) {
+ auto builder = std::make_shared<parquet::ArrowWriterPropertiesBuilder>();
builder->store_schema();
-}
-
-// [[arrow::export]]
-void
parquet___ArrowWriterProperties___Builder__enable_deprecated_int96_timestamps(
- const std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder) {
- builder->enable_deprecated_int96_timestamps();
-}
-
-// [[arrow::export]]
-void
parquet___ArrowWriterProperties___Builder__disable_deprecated_int96_timestamps(
- const std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder) {
- builder->disable_deprecated_int96_timestamps();
-}
-
-// [[arrow::export]]
-void parquet___ArrowWriterProperties___Builder__coerce_timestamps(
- const std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder,
- arrow::TimeUnit::type unit) {
- builder->coerce_timestamps(unit);
-}
-
-// [[arrow::export]]
-void parquet___ArrowWriterProperties___Builder__allow_truncated_timestamps(
- const std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder) {
- builder->allow_truncated_timestamps();
-}
-// [[arrow::export]]
-void parquet___ArrowWriterProperties___Builder__disallow_truncated_timestamps(
- const std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder) {
- builder->disallow_truncated_timestamps();
-}
+ if (allow_truncated_timestamps) {
+ builder->allow_truncated_timestamps();
+ }
+ if (use_deprecated_int96_timestamps) {
+ builder->enable_deprecated_int96_timestamps();
+ }
+ if (timestamp_unit > -1) {
+ // -1 is passed in for NULL/default
+
builder->coerce_timestamps(static_cast<arrow::TimeUnit::type>(timestamp_unit));
+ }
-// [[arrow::export]]
-std::shared_ptr<parquet::ArrowWriterProperties>
-parquet___ArrowWriterProperties___Builder__build(
- const std::shared_ptr<parquet::ArrowWriterPropertiesBuilder>& builder) {
return builder->build();
}
// [[arrow::export]]
-std::shared_ptr<parquet::WriterProperties>
parquet___default_writer_properties() {
- return parquet::default_writer_properties();
-}
-
-// [[arrow::export]]
std::shared_ptr<parquet::WriterPropertiesBuilder>
parquet___WriterProperties___Builder__create() {
return std::make_shared<parquet::WriterPropertiesBuilder>();
@@ -174,58 +140,30 @@ void parquet___WriterProperties___Builder__version(
}
// [[arrow::export]]
-void parquet___ArrowWriterProperties___Builder__default_compression(
- const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
- const arrow::Compression::type& compression) {
- builder->compression(compression);
-}
-
-// [[arrow::export]]
void parquet___ArrowWriterProperties___Builder__set_compressions(
const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
const std::vector<std::string>& paths, const Rcpp::IntegerVector& types) {
- auto n = paths.size();
- for (decltype(n) i = 0; i < n; i++) {
- builder->compression(paths[i],
static_cast<arrow::Compression::type>(types[i]));
+ auto n = types.size();
+ if (n == 1) {
+ builder->compression(static_cast<arrow::Compression::type>(types[0]));
+ } else {
+ for (decltype(n) i = 0; i < n; i++) {
+ builder->compression(paths[i],
static_cast<arrow::Compression::type>(types[i]));
+ }
}
}
// [[arrow::export]]
-void parquet___ArrowWriterProperties___Builder__default_compression_level(
- const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
- int compression_level) {
- builder->compression_level(compression_level);
-}
-
-// [[arrow::export]]
void parquet___ArrowWriterProperties___Builder__set_compression_levels(
const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
const std::vector<std::string>& paths, const Rcpp::IntegerVector& levels) {
- auto n = paths.size();
- for (decltype(n) i = 0; i < n; i++) {
- builder->compression_level(paths[i], levels[i]);
- }
-}
-
-// [[arrow::export]]
-void parquet___ArrowWriterProperties___Builder__default_write_statistics(
- const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
- bool write_statistics) {
- if (write_statistics) {
- builder->enable_statistics();
+ auto n = levels.size();
+ if (n == 1) {
+ builder->compression_level(levels[0]);
} else {
- builder->disable_statistics();
- }
-}
-
-// [[arrow::export]]
-void parquet___ArrowWriterProperties___Builder__default_use_dictionary(
- const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
- bool use_dictionary) {
- if (use_dictionary) {
- builder->enable_dictionary();
- } else {
- builder->disable_dictionary();
+ for (decltype(n) i = 0; i < n; i++) {
+ builder->compression_level(paths[i], levels[i]);
+ }
}
}
@@ -233,13 +171,21 @@ void
parquet___ArrowWriterProperties___Builder__default_use_dictionary(
void parquet___ArrowWriterProperties___Builder__set_use_dictionary(
const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
const std::vector<std::string>& paths, const Rcpp::LogicalVector&
use_dictionary) {
- builder->disable_dictionary();
- auto n = paths.size();
- for (decltype(n) i = 0; i < n; i++) {
- if (use_dictionary[i]) {
- builder->enable_dictionary(paths[i]);
+ auto n = use_dictionary.size();
+ if (n == 1) {
+ if (use_dictionary[0]) {
+ builder->enable_dictionary();
} else {
- builder->disable_dictionary(paths[i]);
+ builder->disable_dictionary();
+ }
+ } else {
+ builder->disable_dictionary();
+ for (decltype(n) i = 0; i < n; i++) {
+ if (use_dictionary[i]) {
+ builder->enable_dictionary(paths[i]);
+ } else {
+ builder->disable_dictionary(paths[i]);
+ }
}
}
}
@@ -248,13 +194,21 @@ void
parquet___ArrowWriterProperties___Builder__set_use_dictionary(
void parquet___ArrowWriterProperties___Builder__set_write_statistics(
const std::shared_ptr<parquet::WriterPropertiesBuilder>& builder,
const std::vector<std::string>& paths, const Rcpp::LogicalVector&
write_statistics) {
- builder->disable_statistics();
- auto n = paths.size();
- for (decltype(n) i = 0; i < n; i++) {
- if (write_statistics[i]) {
- builder->enable_statistics(paths[i]);
+ auto n = write_statistics.size();
+ if (n == 1) {
+ if (write_statistics[0]) {
+ builder->enable_statistics();
} else {
- builder->disable_statistics(paths[i]);
+ builder->disable_statistics();
+ }
+ } else {
+ builder->disable_statistics();
+ for (decltype(n) i = 0; i < n; i++) {
+ if (write_statistics[i]) {
+ builder->enable_statistics(paths[i]);
+ } else {
+ builder->disable_statistics(paths[i]);
+ }
}
}
}
diff --git a/r/tests/testthat/helper-expectation.R
b/r/tests/testthat/helper-expectation.R
index 29fb08a..69597d5 100644
--- a/r/tests/testthat/helper-expectation.R
+++ b/r/tests/testthat/helper-expectation.R
@@ -38,6 +38,12 @@ expect_equivalent <- function(object, expected, ...) {
# expect_equal but for DataTypes, so the error prints better
expect_type_equal <- function(object, expected, ...) {
+ if (is.Array(object)) {
+ object <- object$type
+ }
+ if (is.Array(expected)) {
+ expected <- expected$type
+ }
expect_equal(object, expected, ..., label = object$ToString(),
expected.label = expected$ToString())
}
diff --git a/r/tests/testthat/test-parquet.R b/r/tests/testthat/test-parquet.R
index 23699f5..71484db 100644
--- a/r/tests/testthat/test-parquet.R
+++ b/r/tests/testthat/test-parquet.R
@@ -79,6 +79,15 @@ test_that("write_parquet() handles various use_dictionary=
specs", {
expect_parquet_roundtrip(tab, use_dictionary = TRUE)
expect_parquet_roundtrip(tab, use_dictionary = c(TRUE, FALSE, TRUE))
expect_parquet_roundtrip(tab, use_dictionary = c(x1 = TRUE, x2 = TRUE))
+ expect_error(
+ write_parquet(tab, tempfile(), use_dictionary = c(TRUE, FALSE)),
+ "unsupported use_dictionary= specification"
+ )
+ expect_error(
+ write_parquet(tab, tempfile(), use_dictionary = 12),
+ "is.logical(use_dictionary) is not TRUE",
+ fixed = TRUE
+ )
})
test_that("write_parquet() handles various write_statistics= specs", {
@@ -89,6 +98,19 @@ test_that("write_parquet() handles various write_statistics=
specs", {
expect_parquet_roundtrip(tab, write_statistics = c(x1 = TRUE, x2 = TRUE))
})
+test_that("write_parquet() can truncate timestamps", {
+ tab <- Table$create(x1 = as.POSIXct("2020/06/03 18:00:00", tz = "UTC"))
+ expect_type_equal(tab$x1, timestamp("us", "UTC"))
+
+ tf <- tempfile()
+ on.exit(unlink(tf))
+
+ write_parquet(tab, tf, coerce_timestamps = "ms", allow_truncated_timestamps
= TRUE)
+ new <- read_parquet(tf, as_data_frame = FALSE)
+ expect_type_equal(new$x1, timestamp("ms", "UTC"))
+ expect_equivalent(as.data.frame(tab), as.data.frame(new))
+})
+
test_that("make_valid_version()", {
expect_equal(make_valid_version("1.0"), ParquetVersionType$PARQUET_1_0)
expect_equal(make_valid_version("2.0"), ParquetVersionType$PARQUET_2_0)
