pitrou commented on a change in pull request #8177:
URL: https://github.com/apache/arrow/pull/8177#discussion_r488084615
##########
File path: cpp/src/parquet/level_conversion.cc
##########
@@ -18,176 +18,190 @@
#include <algorithm>
#include <limits>
-#if defined(ARROW_HAVE_BMI2)
-#include <x86intrin.h>
-#endif
+#include "arrow/util/bit_run_reader.h"
#include "arrow/util/bit_util.h"
+#include "arrow/util/cpu_info.h"
#include "arrow/util/logging.h"
#include "parquet/exception.h"
+#include "parquet/level_comparison.h"
+
+#define BMI_RUNTIME_VERSION standard
+#include "parquet/level_conversion_inc.h"
+#undef BMI_RUNTIME_VERSION
namespace parquet {
namespace internal {
namespace {
-inline void CheckLevelRange(const int16_t* levels, int64_t num_levels,
- const int16_t max_expected_level) {
- int16_t min_level = std::numeric_limits<int16_t>::max();
- int16_t max_level = std::numeric_limits<int16_t>::min();
- for (int x = 0; x < num_levels; x++) {
- min_level = std::min(levels[x], min_level);
- max_level = std::max(levels[x], max_level);
- }
- if (ARROW_PREDICT_FALSE(num_levels > 0 &&
- (min_level < 0 || max_level > max_expected_level))) {
- throw ParquetException("definition level exceeds maximum");
- }
-}
-#if !defined(ARROW_HAVE_AVX512)
-
-inline void DefinitionLevelsToBitmapScalar(
- const int16_t* def_levels, int64_t num_def_levels, const int16_t
max_definition_level,
- const int16_t max_repetition_level, int64_t* values_read, int64_t*
null_count,
- uint8_t* valid_bits, int64_t valid_bits_offset) {
- // We assume here that valid_bits is large enough to accommodate the
- // additional definition levels and the ones that have already been written
- ::arrow::internal::BitmapWriter valid_bits_writer(valid_bits,
valid_bits_offset,
- num_def_levels);
-
- // TODO(itaiin): As an interim solution we are splitting the code path here
- // between repeated+flat column reads, and non-repeated+nested reads.
- // Those paths need to be merged in the future
- for (int i = 0; i < num_def_levels; ++i) {
- if (def_levels[i] == max_definition_level) {
+using ::arrow::internal::CpuInfo;
+
+#if !defined(ARROW_HAVE_RUNTIME_BMI2)
+void DefinitionLevelsToBitmapScalar(const int16_t* def_levels, int64_t
num_def_levels,
+ LevelInfo level_info,
+ ValidityBitmapInputOutput* output) {
+ ::arrow::internal::FirstTimeBitmapWriter valid_bits_writer(
+ output->valid_bits,
+ /*start_offset=*/output->valid_bits_offset,
+ /*length=*/num_def_levels);
+ for (int x = 0; x < num_def_levels; x++) {
+ if (def_levels[x] < level_info.repeated_ancestor_def_level) {
Review comment:
Also, in many cases `repeated_ancestor_def_level` would be 0, in which
case we should simply use `GreaterThan`?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
