This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 8f0bcd8 ARROW-4461: [C++] Expose bit map operations that work with
raw pointers
8f0bcd8 is described below
commit 8f0bcd89378caa2539bff14b3196cab8110e1cb3
Author: Micah Kornfield <[email protected]>
AuthorDate: Mon Feb 11 22:01:41 2019 -0600
ARROW-4461: [C++] Expose bit map operations that work with raw pointers
Precursor to fixing the rest of the boolean computer kernels to not
allocate memory.
Author: Micah Kornfield <[email protected]>
Closes #3560 from emkornfield/bin_memory_allocs and squashes the following
commits:
08107f04e <Micah Kornfield> fix format
f824521c3 <Micah Kornfield> address review comments
73f89aaab <Micah Kornfield> fixup method order
f4661fe76 <Micah Kornfield> fixups
ae26a844d <Micah Kornfield> Expose no allocating bit utilities
---
cpp/src/arrow/util/bit-util-test.cc | 39 ++++++++++++++++++++++++++++++++
cpp/src/arrow/util/bit-util.cc | 45 ++++++++++++++++++++++++++++---------
cpp/src/arrow/util/bit-util.h | 39 ++++++++++++++++++++++++++++++++
3 files changed, 113 insertions(+), 10 deletions(-)
diff --git a/cpp/src/arrow/util/bit-util-test.cc
b/cpp/src/arrow/util/bit-util-test.cc
index 93feb5b..71585cd 100644
--- a/cpp/src/arrow/util/bit-util-test.cc
+++ b/cpp/src/arrow/util/bit-util-test.cc
@@ -379,6 +379,10 @@ struct BitmapOperation {
const uint8_t* right, int64_t right_offset, int64_t
length,
int64_t out_offset, std::shared_ptr<Buffer>* out_buffer)
const = 0;
+ virtual Status Call(const uint8_t* left, int64_t left_offset, const uint8_t*
right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
+ uint8_t* out_buffer) const = 0;
+
virtual ~BitmapOperation() = default;
};
@@ -389,6 +393,13 @@ struct BitmapAndOp : public BitmapOperation {
return BitmapAnd(pool, left, left_offset, right, right_offset, length,
out_offset,
out_buffer);
}
+
+ Status Call(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
+ uint8_t* out_buffer) const override {
+ BitmapAnd(left, left_offset, right, right_offset, length, out_offset,
out_buffer);
+ return Status::OK();
+ }
};
struct BitmapOrOp : public BitmapOperation {
@@ -398,6 +409,13 @@ struct BitmapOrOp : public BitmapOperation {
return BitmapOr(pool, left, left_offset, right, right_offset, length,
out_offset,
out_buffer);
}
+
+ Status Call(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
+ uint8_t* out_buffer) const override {
+ BitmapOr(left, left_offset, right, right_offset, length, out_offset,
out_buffer);
+ return Status::OK();
+ }
};
struct BitmapXorOp : public BitmapOperation {
@@ -407,6 +425,13 @@ struct BitmapXorOp : public BitmapOperation {
return BitmapXor(pool, left, left_offset, right, right_offset, length,
out_offset,
out_buffer);
}
+
+ Status Call(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
+ uint8_t* out_buffer) const override {
+ BitmapXor(left, left_offset, right, right_offset, length, out_offset,
out_buffer);
+ return Status::OK();
+ }
};
class BitmapOp : public TestBase {
@@ -427,6 +452,13 @@ class BitmapOp : public TestBase {
&out));
auto reader = internal::BitmapReader(out->mutable_data(),
out_offset, length);
ASSERT_READER_VALUES(reader, result_bits);
+
+ // Clear out buffer and try non-allocating version
+ std::memset(out->mutable_data(), 0, out->size());
+ ASSERT_OK(op.Call(left->mutable_data(), left_offset,
right->mutable_data(),
+ right_offset, length, out_offset,
out->mutable_data()));
+ reader = internal::BitmapReader(out->mutable_data(), out_offset,
length);
+ ASSERT_READER_VALUES(reader, result_bits);
}
}
}
@@ -451,6 +483,13 @@ class BitmapOp : public TestBase {
&out));
auto reader = internal::BitmapReader(out->mutable_data(),
out_offset, length);
ASSERT_READER_VALUES(reader, result_bits);
+
+ // Clear out buffer and try non-allocating version
+ std::memset(out->mutable_data(), 0, out->size());
+ ASSERT_OK(op.Call(left->mutable_data(), left_offset,
right->mutable_data(),
+ right_offset, length, out_offset,
out->mutable_data()));
+ reader = internal::BitmapReader(out->mutable_data(), out_offset,
length);
+ ASSERT_READER_VALUES(reader, result_bits);
}
}
}
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 862b1fd..2e7bf2f 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -296,21 +296,28 @@ void UnalignedBitmapOp(const uint8_t* left, int64_t
left_offset, const uint8_t*
}
template <typename BitOp, typename LogicalOp>
-Status BitmapOp(MemoryPool* pool, const uint8_t* left, int64_t left_offset,
- const uint8_t* right, int64_t right_offset, int64_t length,
- int64_t out_offset, std::shared_ptr<Buffer>* out_buffer) {
+void BitmapOp(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
uint8_t* dest) {
if ((out_offset % 8 == left_offset % 8) && (out_offset % 8 == right_offset %
8)) {
// Fast case: can use bytewise AND
- const int64_t phys_bits = length + out_offset;
- RETURN_NOT_OK(AllocateEmptyBitmap(pool, phys_bits, out_buffer));
- AlignedBitmapOp<BitOp>(left, left_offset, right, right_offset,
- (*out_buffer)->mutable_data(), out_offset, length);
+ AlignedBitmapOp<BitOp>(left, left_offset, right, right_offset, dest,
out_offset,
+ length);
} else {
// Unaligned
- RETURN_NOT_OK(AllocateEmptyBitmap(pool, length + out_offset, out_buffer));
- UnalignedBitmapOp<LogicalOp>(left, left_offset, right, right_offset,
- (*out_buffer)->mutable_data(), out_offset,
length);
+ UnalignedBitmapOp<LogicalOp>(left, left_offset, right, right_offset, dest,
out_offset,
+ length);
}
+}
+
+template <typename BitOp, typename LogicalOp>
+Status BitmapOp(MemoryPool* pool, const uint8_t* left, int64_t left_offset,
+ const uint8_t* right, int64_t right_offset, int64_t length,
+ int64_t out_offset, std::shared_ptr<Buffer>* out_buffer) {
+ const int64_t phys_bits = length + out_offset;
+ RETURN_NOT_OK(AllocateEmptyBitmap(pool, phys_bits, out_buffer));
+ uint8_t* out = (*out_buffer)->mutable_data();
+ BitmapOp<BitOp, LogicalOp>(left, left_offset, right, right_offset, length,
out_offset,
+ out);
return Status::OK();
}
@@ -323,6 +330,12 @@ Status BitmapAnd(MemoryPool* pool, const uint8_t* left,
int64_t left_offset,
pool, left, left_offset, right, right_offset, length, out_offset,
out_buffer);
}
+void BitmapAnd(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
uint8_t* out) {
+ BitmapOp<std::bit_and<uint8_t>, std::logical_and<bool>>(
+ left, left_offset, right, right_offset, length, out_offset, out);
+}
+
Status BitmapOr(MemoryPool* pool, const uint8_t* left, int64_t left_offset,
const uint8_t* right, int64_t right_offset, int64_t length,
int64_t out_offset, std::shared_ptr<Buffer>* out_buffer) {
@@ -330,6 +343,12 @@ Status BitmapOr(MemoryPool* pool, const uint8_t* left,
int64_t left_offset,
pool, left, left_offset, right, right_offset, length, out_offset,
out_buffer);
}
+void BitmapOr(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
uint8_t* out) {
+ BitmapOp<std::bit_or<uint8_t>, std::logical_or<bool>>(
+ left, left_offset, right, right_offset, length, out_offset, out);
+}
+
Status BitmapXor(MemoryPool* pool, const uint8_t* left, int64_t left_offset,
const uint8_t* right, int64_t right_offset, int64_t length,
int64_t out_offset, std::shared_ptr<Buffer>* out_buffer) {
@@ -337,5 +356,11 @@ Status BitmapXor(MemoryPool* pool, const uint8_t* left,
int64_t left_offset,
pool, left, left_offset, right, right_offset, length, out_offset,
out_buffer);
}
+void BitmapXor(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
uint8_t* out) {
+ BitmapOp<std::bit_xor<uint8_t>, std::bit_xor<bool>>(
+ left, left_offset, right, right_offset, length, out_offset, out);
+}
+
} // namespace internal
} // namespace arrow
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 0e4a864..888b79d 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -738,21 +738,60 @@ ARROW_EXPORT
bool BitmapEquals(const uint8_t* left, int64_t left_offset, const uint8_t*
right,
int64_t right_offset, int64_t bit_length);
+/// \brief Do a "bitmap and" on right and left buffers starting at
+/// their respective bit-offsets for the given bit-length and put
+/// the results in out_buffer starting at the given bit-offset.
+///
+/// out_buffer will be allocated and initialized to zeros using pool before
+/// the operation.
ARROW_EXPORT
Status BitmapAnd(MemoryPool* pool, const uint8_t* left, int64_t left_offset,
const uint8_t* right, int64_t right_offset, int64_t length,
int64_t out_offset, std::shared_ptr<Buffer>* out_buffer);
+/// \brief Do a "bitmap and" on right and left buffers starting at
+/// their respective bit-offsets for the given bit-length and put
+/// the results in out starting at the given bit-offset.
+ARROW_EXPORT
+void BitmapAnd(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
uint8_t* out);
+
+/// \brief Do a "bitmap or" for the given bit length on right and left buffers
+/// starting at their respective bit-offsets and put the results in out_buffer
+/// starting at the given bit-offset.
+///
+/// out_buffer will be allocated and initialized to zeros using pool before
+/// the operation.
ARROW_EXPORT
Status BitmapOr(MemoryPool* pool, const uint8_t* left, int64_t left_offset,
const uint8_t* right, int64_t right_offset, int64_t length,
int64_t out_offset, std::shared_ptr<Buffer>* out_buffer);
+/// \brief Do a "bitmap or" for the given bit length on right and left buffers
+/// starting at their respective bit-offsets and put the results in out
+/// starting at the given bit-offset.
+ARROW_EXPORT
+void BitmapOr(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
uint8_t* out);
+
+/// \brief Do a "bitmap xor" for the given bit-length on right and left
+/// buffers starting at their respective bit-offsets and put the results in
+/// out_buffer starting at the given bit offset.
+///
+/// out_buffer will be allocated and initialized to zeros using pool before
+/// the operation.
ARROW_EXPORT
Status BitmapXor(MemoryPool* pool, const uint8_t* left, int64_t left_offset,
const uint8_t* right, int64_t right_offset, int64_t length,
int64_t out_offset, std::shared_ptr<Buffer>* out_buffer);
+/// \brief Do a "bitmap xor" for the given bit-length on right and left
+/// buffers starting at their respective bit-offsets and put the results in
+/// out starting at the given bit offset.
+ARROW_EXPORT
+void BitmapXor(const uint8_t* left, int64_t left_offset, const uint8_t* right,
+ int64_t right_offset, int64_t length, int64_t out_offset,
uint8_t* out);
+
} // namespace internal
} // namespace arrow
