nealrichardson commented on a change in pull request #7514:
URL: https://github.com/apache/arrow/pull/7514#discussion_r446706140
##########
File path: r/src/array_from_vector.cpp
##########
@@ -918,6 +923,97 @@ class Time64Converter : public TimeConverter<Time64Type> {
}
};
+template <typename Builder>
+class BinaryVectorConverter : public VectorConverter {
+ public:
+ ~BinaryVectorConverter() {}
+
+ Status Init(ArrayBuilder* builder) {
+ typed_builder_ = checked_cast<Builder*>(builder);
+ return Status::OK();
+ }
+
+ Status Ingest(SEXP obj) {
+ ARROW_RETURN_IF(TYPEOF(obj) != VECSXP, Status::RError("Expecting a list"));
+ R_xlen_t n = XLENGTH(obj);
+
+ // Reserve enough space before appending
+ int64_t size = 0;
+ for (R_xlen_t i = 0; i < n; i++) {
+ SEXP obj_i = VECTOR_ELT(obj, i);
+ if (!Rf_isNull(obj_i)) {
+ ARROW_RETURN_IF(TYPEOF(obj_i) != RAWSXP,
+ Status::RError("Expecting a raw vector"));
+ size += XLENGTH(obj_i);
+ }
+ }
+ RETURN_NOT_OK(typed_builder_->Reserve(size));
+
+ // append
+ for (R_xlen_t i = 0; i < n; i++) {
+ SEXP obj_i = VECTOR_ELT(obj, i);
+ if (Rf_isNull(obj_i)) {
+ RETURN_NOT_OK(typed_builder_->AppendNull());
+ } else {
+ RETURN_NOT_OK(typed_builder_->Append(RAW(obj_i), XLENGTH(obj_i)));
+ }
+ }
+ return Status::OK();
+ }
+
+ Status GetResult(std::shared_ptr<arrow::Array>* result) {
+ return typed_builder_->Finish(result);
+ }
+
+ private:
+ Builder* typed_builder_;
+};
+
+template <typename Builder>
+class StringVectorConverter : public VectorConverter {
+ public:
+ ~StringVectorConverter() {}
+
+ Status Init(ArrayBuilder* builder) {
+ typed_builder_ = checked_cast<Builder*>(builder);
+ return Status::OK();
+ }
+
+ Status Ingest(SEXP obj) {
+ ARROW_RETURN_IF(TYPEOF(obj) != STRSXP,
+ Status::RError("Expecting a character vector"));
+ R_xlen_t n = XLENGTH(obj);
+
+ // Reserve enough space before appending
+ int64_t size = 0;
+ for (R_xlen_t i = 0; i < n; i++) {
+ SEXP string_i = STRING_ELT(obj, i);
+ if (string_i != NA_STRING) {
+ size += XLENGTH(string_i);
+ }
+ }
+ RETURN_NOT_OK(typed_builder_->Reserve(size));
+
+ // append
+ for (R_xlen_t i = 0; i < n; i++) {
+ SEXP string_i = STRING_ELT(obj, i);
+ if (string_i == NA_STRING) {
+ RETURN_NOT_OK(typed_builder_->AppendNull());
+ } else {
+ RETURN_NOT_OK(typed_builder_->Append(CHAR(string_i),
XLENGTH(string_i)));
Review comment:
This new builder is breaking the (new) UTF-8 tests. The previous
converter code is
https://github.com/apache/arrow/blob/master/r/src/array_from_vector.cpp#L147-L171
and it is apparently no longer being called.
I wonder if this whole code block isn't possible right now as is. The
"Reserve enough space before appending" block would also need to convert to
UTF-8 in order to get the size right, and I wonder if converting/asserting
everything to UTF-8 twice would outweigh the benefits of Reserving space. Or
maybe we can take the size as is and overcommit bytes?
Tangentially related, would a "reserve" check like this be the way to solve
https://issues.apache.org/jira/browse/ARROW-3308, where we need to switch to
Large types if there's more than 2GB?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
