dalingmeng commented on code in PR #340:
URL: https://github.com/apache/paimon-cpp/pull/340#discussion_r4023185390
##########
src/paimon/common/utils/arrow/arrow_utils.cpp:
##########
@@ -90,6 +89,153 @@ bool HasUndeclaredDictionaryChild(const
std::shared_ptr<arrow::DataType>& type,
return false;
}
+// Positions in the current array which remain visible through every nullable
ancestor.
+struct VisibleRange {
+ int64_t offset;
+ int64_t length;
+};
+
+using VisibleRanges = std::vector<VisibleRange>;
+
+void AppendVisibleRange(int64_t offset, int64_t length, VisibleRanges* ranges)
{
+ if (length == 0) {
+ return;
+ }
+ if (!ranges->empty()) {
+ VisibleRange& last = ranges->back();
+ if (last.offset + last.length == offset) {
+ last.length += length;
+ return;
+ }
+ }
+ ranges->push_back({offset, length});
+}
+
+Status NullabilityMismatch(const arrow::Field& field) {
+ return Status::Invalid(fmt::format(
+ "CheckNullabilityMatch failed, field {} not nullable while data have
null value",
+ field.name()));
+}
+
+Result<VisibleRanges> IntersectWithValidity(const arrow::Array& array,
+ const VisibleRanges&
visible_ranges,
+ const arrow::Field& field) {
+ const bool nullable = field.nullable();
+ if (visible_ranges.empty()) {
+ return VisibleRanges{};
+ }
+
+ const int64_t null_count = array.null_count();
+ if (null_count == 0) {
+ return visible_ranges;
+ }
+ if (null_count == array.length()) {
+ if (!nullable) {
+ return NullabilityMismatch(field);
+ }
+ return VisibleRanges{};
+ }
+
+ VisibleRanges valid_ranges;
+ for (const VisibleRange& range : visible_ranges) {
Review Comment:
Non-blocking / optional (up to you):IntersectWithValidity scans visible
ranges bit-by-bit via array.IsValid(i). Correct as-is — just a perf note:
Arrow's arrow::internal::SetBitRunReader (arrow/util/bit_run_reader.h) scans
the validity bitmap word-by-word and yields runs of set bits directly, skipping
all-ones words instead of testing each bit. Since validity bitmaps are usually
mostly-ones and this runs on the main write path, it could help on large arrays
— and its output is exactly the "valid ranges" we're building here.
VisibleRanges valid_ranges;
const uint8_t* bitmap = array.null_bitmap_data();
for (const VisibleRange& range : visible_ranges) {
const int64_t base = array.offset() + range.offset; // IsValid(i)
uses logical index; bitmap bit is array.offset()+i
arrow::internal::SetBitRunReader reader(bitmap, base, range.length);
int64_t covered = 0;
for (auto run = reader.NextRun(); !run.AtEnd(); run =
reader.NextRun()) {
AppendVisibleRange(range.offset + run.position, run.length,
&valid_ranges); // position is relative to base
covered += run.length;
}
if (!nullable && covered != range.length) { // a visible-range
null; can't use global null_count (nulls may sit under a null parent)
return NullabilityMismatch(field);
}
}
return valid_ranges;
Note this would be the first use of bit_run_reader.h in the repo, so feel
free to skip if you'd rather not introduce a new pattern for a
micro-optimization.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
