kiszk commented on a change in pull request #6985: URL: https://github.com/apache/arrow/pull/6985#discussion_r413960907
########## File path: cpp/src/parquet/level_conversion.h ########## @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <cstdint> +#include "arrow/util/bit_util.h" + +#if defined(ARROW_HAVE_BMI2) +#include "x86intrin.h" +#endif + +namespace parquet { +namespace internal { +// These APIs are likely to be revised as part of ARROW-8494 to reduce duplicate code. +// They currently represent minimal functionality for vectorized computation of definition +// levels. + +/// Builds a bitmap by applying predicate to the level vector provided. +/// +/// \param[in] levels Rep or def level array. +/// \param[in] num_levels The number of levels to process (must be [0, 64]) +/// \param[in] predicate The predicate to apply (must have the signature `bool +/// predicate(int16_t)`. +/// \returns The bitmap using least significant "bit" ordering. +/// +/// N.B. Correct byte ordering is dependent on little-endian architectures. +/// +template <typename Predicate> +uint64_t LevelsToBitmap(const int16_t* levels, int64_t num_levels, Predicate predicate) { + // Both clang and GCC can vectorize this automatically with AVX2. + uint64_t mask = 0; + for (int x = 0; x < num_levels; x++) { + mask |= static_cast<int64_t>(predicate(levels[x]) ? 1 : 0) << x; + } + return mask; +} + +/// Builds a bitmap where each set bit indicates the correspond level is greater +/// than rhs. +static inline int64_t GreaterThanBitmap(const int16_t* levels, int64_t num_levels, + int16_t rhs) { + return LevelsToBitmap(levels, num_levels, [&](int16_t value) { return value > rhs; }); +} + +/// Append bits number_of_bits from new_bits to valid_bits and valid_bits_offset. +/// +/// \param[in] new_bits The zero-padded bitmap to append. +/// \param[in] number_of_bits The number of bits to append from new_bits. +/// \param[in] valid_bits_length The number of bytes allocated in valid_bits. +/// \param[in] valid_bits_offset The bit-offset at which to start appending new bits. +/// \param[in,out] valid_bits The validity bitmap to append to. +/// \returns The new bit offset inside of valid_bits. +static inline int64_t AppendBitmap(uint64_t new_bits, int64_t number_of_bits, + int64_t valid_bits_length, int64_t valid_bits_offset, + uint8_t* valid_bits) { + // Selection masks to retrieve all low order bits for each bytes. + constexpr uint64_t kLsbSelectionMasks[] = { + 0, // unused. + 0x0101010101010101, + 0x0303030303030303, + 0x0707070707070707, + 0x0F0F0F0F0F0F0F0F, + 0x1F1F1F1F1F1F1F1F, + 0x3F3F3F3F3F3F3F3F, + 0x7F7F7F7F7F7F7F7F, + }; + int64_t valid_byte_offset = valid_bits_offset / 8; + int64_t bit_offset = valid_bits_offset % 8; + + int64_t new_offset = valid_bits_offset + number_of_bits; + union ByteAddressableBitmap { Review comment: Why do we need to use union? `bytes[]` is used only at line 106. It looks simple to explicitly extract the lowest 8bit instead of using union. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
