wesm commented on code in PR #13364:
URL: https://github.com/apache/arrow/pull/13364#discussion_r899500164
##########
cpp/src/arrow/array/data.cc:
##########
@@ -128,6 +131,141 @@ int64_t ArrayData::GetNullCount() const {
return precomputed;
}
+// ----------------------------------------------------------------------
+// Methods for ArraySpan
+
+void ArraySpan::SetMembers(const ArrayData& data) {
+ this->type = data.type.get();
+ this->length = data.length;
+ this->null_count = data.null_count.load();
+ this->offset = data.offset;
+
+ for (int i = 0; i < static_cast<int>(data.buffers.size()); ++i) {
+ const std::shared_ptr<Buffer>& buffer = data.buffers[i];
+ // It is the invoker-of-kernels's responsibility to ensure that
+ // const buffers are not written to accidentally.
+ if (buffer) {
+ SetBuffer(i, buffer);
+ } else {
+ ClearBuffer(i);
+ }
+ }
+
+ // Makes sure any other buffers are seen as null / non-existent
+ for (int i = static_cast<int>(data.buffers.size()); i < 3; ++i) {
+ ClearBuffer(i);
+ }
+
+ if (this->type->id() == Type::DICTIONARY) {
+ this->child_data.resize(1);
+ this->child_data[0].SetMembers(*data.dictionary);
+ } else {
+ this->child_data.resize(data.child_data.size());
+ for (size_t child_index = 0; child_index < data.child_data.size();
++child_index) {
+ this->child_data[child_index].SetMembers(*data.child_data[child_index]);
+ }
+ }
+}
+
+void ArraySpan::FillFromScalar(const Scalar& value) {
+ static const uint8_t kValidByte = 0x01;
+ static const uint8_t kNullByte = 0x00;
+
+ this->type = value.type.get();
+ this->length = 1;
+
+ // Populate null count and validity bitmap
+ this->null_count = value.is_valid ? 0 : 1;
+ this->buffers[0].data = const_cast<uint8_t*>(value.is_valid ? &kValidByte :
&kNullByte);
+ this->buffers[0].size = 1;
+
+ if (is_primitive(value.type->id())) {
+ const auto& scalar =
+ internal::checked_cast<const internal::PrimitiveScalarBase&>(value);
+ const uint8_t* scalar_data = reinterpret_cast<const
uint8_t*>(scalar.view().data());
+ this->buffers[1].data = const_cast<uint8_t*>(scalar_data);
+ this->buffers[1].size = scalar.type->byte_width();
+ } else {
+ // TODO(wesm): implement for other types
+ DCHECK(false) << "need to implement for other types";
+ }
+}
+
+int64_t ArraySpan::GetNullCount() const {
+ int64_t precomputed = this->null_count;
+ if (ARROW_PREDICT_FALSE(precomputed == kUnknownNullCount)) {
+ if (this->buffers[0].data != nullptr) {
+ precomputed =
+ this->length - CountSetBits(this->buffers[0].data, this->offset,
this->length);
+ } else {
+ precomputed = 0;
+ }
+ this->null_count = precomputed;
+ }
+ return precomputed;
+}
+
+int GetNumBuffers(const DataType& type) {
+ switch (type.id()) {
+ case Type::NA:
+ return 0;
+ case Type::STRUCT:
+ case Type::FIXED_SIZE_LIST:
+ return 1;
+ case Type::BINARY:
+ case Type::LARGE_BINARY:
+ case Type::STRING:
+ case Type::LARGE_STRING:
+ case Type::DENSE_UNION:
+ return 3;
+ case Type::EXTENSION:
+ // The number of buffers depends on the storage type
+ return GetNumBuffers(
+ *internal::checked_cast<const ExtensionType&>(type).storage_type());
+ default:
+ // Everything else has 2 buffers
+ return 2;
+ }
+}
+
+int ArraySpan::num_buffers() const { return GetNumBuffers(*this->type); }
+
+std::shared_ptr<ArrayData> ArraySpan::ToArrayData() const {
+ auto result = std::make_shared<ArrayData>(this->type->Copy(), this->length,
+ kUnknownNullCount, this->offset);
Review Comment:
fixing in follow up RP
##########
cpp/src/arrow/array/data.cc:
##########
@@ -128,6 +131,141 @@ int64_t ArrayData::GetNullCount() const {
return precomputed;
}
+// ----------------------------------------------------------------------
+// Methods for ArraySpan
+
+void ArraySpan::SetMembers(const ArrayData& data) {
+ this->type = data.type.get();
+ this->length = data.length;
+ this->null_count = data.null_count.load();
+ this->offset = data.offset;
+
+ for (int i = 0; i < static_cast<int>(data.buffers.size()); ++i) {
+ const std::shared_ptr<Buffer>& buffer = data.buffers[i];
+ // It is the invoker-of-kernels's responsibility to ensure that
+ // const buffers are not written to accidentally.
+ if (buffer) {
+ SetBuffer(i, buffer);
+ } else {
+ ClearBuffer(i);
+ }
+ }
+
+ // Makes sure any other buffers are seen as null / non-existent
+ for (int i = static_cast<int>(data.buffers.size()); i < 3; ++i) {
+ ClearBuffer(i);
+ }
+
+ if (this->type->id() == Type::DICTIONARY) {
+ this->child_data.resize(1);
+ this->child_data[0].SetMembers(*data.dictionary);
+ } else {
+ this->child_data.resize(data.child_data.size());
+ for (size_t child_index = 0; child_index < data.child_data.size();
++child_index) {
+ this->child_data[child_index].SetMembers(*data.child_data[child_index]);
+ }
+ }
+}
+
+void ArraySpan::FillFromScalar(const Scalar& value) {
+ static const uint8_t kValidByte = 0x01;
+ static const uint8_t kNullByte = 0x00;
+
+ this->type = value.type.get();
+ this->length = 1;
+
+ // Populate null count and validity bitmap
+ this->null_count = value.is_valid ? 0 : 1;
+ this->buffers[0].data = const_cast<uint8_t*>(value.is_valid ? &kValidByte :
&kNullByte);
+ this->buffers[0].size = 1;
+
+ if (is_primitive(value.type->id())) {
+ const auto& scalar =
+ internal::checked_cast<const internal::PrimitiveScalarBase&>(value);
+ const uint8_t* scalar_data = reinterpret_cast<const
uint8_t*>(scalar.view().data());
+ this->buffers[1].data = const_cast<uint8_t*>(scalar_data);
+ this->buffers[1].size = scalar.type->byte_width();
+ } else {
+ // TODO(wesm): implement for other types
+ DCHECK(false) << "need to implement for other types";
+ }
+}
+
+int64_t ArraySpan::GetNullCount() const {
+ int64_t precomputed = this->null_count;
+ if (ARROW_PREDICT_FALSE(precomputed == kUnknownNullCount)) {
+ if (this->buffers[0].data != nullptr) {
+ precomputed =
+ this->length - CountSetBits(this->buffers[0].data, this->offset,
this->length);
+ } else {
+ precomputed = 0;
+ }
+ this->null_count = precomputed;
+ }
+ return precomputed;
+}
+
+int GetNumBuffers(const DataType& type) {
+ switch (type.id()) {
+ case Type::NA:
+ return 0;
+ case Type::STRUCT:
+ case Type::FIXED_SIZE_LIST:
+ return 1;
+ case Type::BINARY:
+ case Type::LARGE_BINARY:
+ case Type::STRING:
+ case Type::LARGE_STRING:
+ case Type::DENSE_UNION:
+ return 3;
+ case Type::EXTENSION:
+ // The number of buffers depends on the storage type
+ return GetNumBuffers(
+ *internal::checked_cast<const ExtensionType&>(type).storage_type());
+ default:
+ // Everything else has 2 buffers
+ return 2;
+ }
+}
+
+int ArraySpan::num_buffers() const { return GetNumBuffers(*this->type); }
+
+std::shared_ptr<ArrayData> ArraySpan::ToArrayData() const {
+ auto result = std::make_shared<ArrayData>(this->type->Copy(), this->length,
+ kUnknownNullCount, this->offset);
Review Comment:
fixing in follow up PR
##########
cpp/src/arrow/array/data.h:
##########
@@ -242,6 +245,131 @@ struct ARROW_EXPORT ArrayData {
std::shared_ptr<ArrayData> dictionary;
};
+/// \brief A non-owning Buffer reference
+struct ARROW_EXPORT BufferSpan {
+ // It is the user of this class's responsibility to ensure that
+ // buffers that were const originally are not written to
+ // accidentally.
+ uint8_t* data = NULLPTR;
+ int64_t size = 0;
+ // Pointer back to buffer that owns this memory
+ const std::shared_ptr<Buffer>* owner = NULLPTR;
+};
+
+/// \brief EXPERIMENTAL: A non-owning ArrayData reference that is cheaply
+/// copyable and does not contain any shared_ptr objects. Do not use in public
+/// APIs aside from compute kernels for now
+struct ARROW_EXPORT ArraySpan {
+ const DataType* type = NULLPTR;
+ int64_t length = 0;
+ mutable int64_t null_count = kUnknownNullCount;
+ int64_t offset = 0;
+ BufferSpan buffers[3];
+
+ ArraySpan() = default;
+
+ explicit ArraySpan(const DataType* type, int64_t length) : type(type),
length(length) {}
+
+ ArraySpan(const ArrayData& data) { // NOLINT implicit conversion
+ SetMembers(data);
+ }
+ ArraySpan(const Scalar& data) { // NOLINT implicit converstion
+ FillFromScalar(data);
+ }
+
+ /// If dictionary-encoded, put dictionary in the first entry
+ std::vector<ArraySpan> child_data;
+
+ /// \brief Populate ArraySpan to look like an array of length 1 pointing at
+ /// the data members of a Scalar value
+ void FillFromScalar(const Scalar& value);
+
+ void SetMembers(const ArrayData& data);
+
+ void SetBuffer(int index, const std::shared_ptr<Buffer>& buffer) {
+ this->buffers[index].data = const_cast<uint8_t*>(buffer->data());
+ this->buffers[index].size = buffer->size();
+ this->buffers[index].owner = &buffer;
+ }
+
+ void ClearBuffer(int index) {
+ this->buffers[index].data = NULLPTR;
+ this->buffers[index].size = 0;
+ this->buffers[index].owner = NULLPTR;
+ }
+
+ const ArraySpan& dictionary() const { return child_data[0]; }
+
+ /// \brief Return the number of buffers (out of 3) that are used to
+ /// constitute this array
+ int num_buffers() const;
+
+ // Access a buffer's data as a typed C pointer
+ template <typename T>
+ inline T* GetValues(int i, int64_t absolute_offset) {
+ return reinterpret_cast<T*>(buffers[i].data) + absolute_offset;
+ }
+
+ template <typename T>
+ inline T* GetValues(int i) {
+ return GetValues<T>(i, this->offset);
+ }
+
+ // Access a buffer's data as a typed C pointer
+ template <typename T>
+ inline const T* GetValues(int i, int64_t absolute_offset) const {
+ return reinterpret_cast<const T*>(buffers[i].data) + absolute_offset;
+ }
+
+ template <typename T>
+ inline const T* GetValues(int i) const {
+ return GetValues<T>(i, this->offset);
+ }
+
+ bool IsNull(int64_t i) const {
+ return ((this->buffers[0].data != NULLPTR)
+ ? !bit_util::GetBit(this->buffers[0].data, i + this->offset)
+ : this->null_count == this->length);
+ }
+
+ bool IsValid(int64_t i) const {
+ return ((this->buffers[0].data != NULLPTR)
+ ? bit_util::GetBit(this->buffers[0].data, i + this->offset)
+ : this->null_count != this->length);
+ }
Review Comment:
https://issues.apache.org/jira/browse/ARROW-16845
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
