This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 6f6051b7501 GH-50678: [C++][Parquet] Remove unused member
`null_slot_usage` in struct `LevelInfo` (#50679)
6f6051b7501 is described below
commit 6f6051b7501c4ef5421e1b6afa94bc75572571a6
Author: Zehua Zou <[email protected]>
AuthorDate: Tue Jul 28 16:52:49 2026 +0800
GH-50678: [C++][Parquet] Remove unused member `null_slot_usage` in struct
`LevelInfo` (#50679)
### Rationale for this change
Remove unused member `null_slot_usage` in struct `LevelInfo` which is in
the `parquet::internal` namespace.
### What changes are included in this PR?
Remove member `null_slot_usage` in struct `LevelInfo`.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
No.
* GitHub Issue: #50678
Authored-by: Zehua Zou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
cpp/src/parquet/arrow/arrow_schema_test.cc | 243 +++++++++++++----------------
cpp/src/parquet/level_conversion.cc | 11 +-
cpp/src/parquet/level_conversion.h | 25 +--
3 files changed, 115 insertions(+), 164 deletions(-)
diff --git a/cpp/src/parquet/arrow/arrow_schema_test.cc
b/cpp/src/parquet/arrow/arrow_schema_test.cc
index 7a7b5a33693..894f6890028 100644
--- a/cpp/src/parquet/arrow/arrow_schema_test.cc
+++ b/cpp/src/parquet/arrow/arrow_schema_test.cc
@@ -2214,13 +2214,12 @@ TEST_F(TestLevels, TestPrimitive) {
PrimitiveNode::Make("node_name", Repetition::REQUIRED,
ParquetType::BOOLEAN));
ASSERT_OK_AND_ASSIGN(std::deque<LevelInfo> levels,
RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
- EXPECT_THAT(levels, ElementsAre(LevelInfo{/*null_slot_usage=*/1,
- /*def_level=*/0, /*rep_level=*/0,
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/0, /*rep_level=*/0,
/*ancestor_list_def_level*/ 0}));
SetParquetSchema(
PrimitiveNode::Make("node_name", Repetition::OPTIONAL,
ParquetType::BOOLEAN));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
- EXPECT_THAT(levels, ElementsAre(LevelInfo{/*null_slot_usage=*/1,
/*def_level=*/1,
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/1,
/*rep_level=*/0,
/*ancestor_list_def_level*/ 0}));
@@ -2228,12 +2227,11 @@ TEST_F(TestLevels, TestPrimitive) {
SetParquetSchema(
PrimitiveNode::Make("node_name", Repetition::REPEATED,
ParquetType::BOOLEAN));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0}, // List Field
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 1})); // primitive
field
+ EXPECT_THAT(levels,
+ ElementsAre(LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0}, // List
Field
+ LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 1})); //
primitive field
}
TEST_F(TestLevels, TestMaps) {
@@ -2248,24 +2246,20 @@ TEST_F(TestLevels, TestMaps) {
GroupNode::Make("my_map", Repetition::OPTIONAL, {list},
LogicalType::Map()));
ASSERT_OK_AND_ASSIGN(std::deque<LevelInfo> levels,
RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 2},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 2}));
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/2, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/2, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 2},
+ LevelInfo{/*def_level=*/2, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 2}));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/1));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 2},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 2}));
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/2, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/2, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 2},
+ LevelInfo{/*def_level=*/3, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 2}));
// single column map.
key = PrimitiveNode::Make("key", Repetition::REQUIRED,
ParquetType::BYTE_ARRAY,
@@ -2276,12 +2270,10 @@ TEST_F(TestLevels, TestMaps) {
GroupNode::Make("my_set", Repetition::REQUIRED, {list},
LogicalType::Map()));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 1}));
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 1}));
}
TEST_F(TestLevels, TestSimpleGroups) {
@@ -2293,14 +2285,12 @@ TEST_F(TestLevels, TestSimpleGroups) {
{PrimitiveNode::Make("inner", Repetition::REQUIRED,
ParquetType::BOOLEAN)})}));
ASSERT_OK_AND_ASSIGN(std::deque<LevelInfo> levels,
RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/0,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/0,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/0,
- /*ancestor_list_def_level*/ 0}));
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/1, /*rep_level=*/0,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/2, /*rep_level=*/0,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/2, /*rep_level=*/0,
+ /*ancestor_list_def_level*/ 0}));
// Arrow schema: struct(child: struct(inner: boolean ))
SetParquetSchema(GroupNode::Make(
@@ -2309,14 +2299,12 @@ TEST_F(TestLevels, TestSimpleGroups) {
"child", Repetition::OPTIONAL,
{PrimitiveNode::Make("inner", Repetition::OPTIONAL,
ParquetType::BOOLEAN)})}));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/0,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/0,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3,
/*rep_level=*/0,
- /*ancestor_list_def_level*/ 0}));
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/1, /*rep_level=*/0,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/2, /*rep_level=*/0,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/3, /*rep_level=*/0,
+ /*ancestor_list_def_level*/ 0}));
// Arrow schema: struct(child: struct(inner: boolean)) not null
SetParquetSchema(GroupNode::Make(
@@ -2325,14 +2313,12 @@ TEST_F(TestLevels, TestSimpleGroups) {
"child", Repetition::OPTIONAL,
{PrimitiveNode::Make("inner", Repetition::OPTIONAL,
ParquetType::BOOLEAN)})}));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/0,
/*rep_level=*/0,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/0,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/0,
- /*ancestor_list_def_level*/ 0}));
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/0, /*rep_level=*/0,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/1, /*rep_level=*/0,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/2, /*rep_level=*/0,
+ /*ancestor_list_def_level*/ 0}));
}
TEST_F(TestLevels, TestRepeatedGroups) {
@@ -2346,12 +2332,10 @@ TEST_F(TestLevels, TestRepeatedGroups) {
ASSERT_OK_AND_ASSIGN(std::deque<LevelInfo> levels,
RootToTreeLeafLevels(*manifest_, /*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 2}));
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/2, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/3, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 2}));
// Arrow schema: list(bool) not null
SetParquetSchema(GroupNode::Make(
@@ -2362,12 +2346,10 @@ TEST_F(TestLevels, TestRepeatedGroups) {
LogicalType::List()));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 1}));
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/2, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 1}));
// Arrow schema: list(bool not null)
SetParquetSchema(GroupNode::Make(
@@ -2378,12 +2360,10 @@ TEST_F(TestLevels, TestRepeatedGroups) {
LogicalType::List()));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 2}));
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/2, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/2, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 2}));
// Arrow schema: list(bool not null) not null
SetParquetSchema(GroupNode::Make(
@@ -2394,12 +2374,10 @@ TEST_F(TestLevels, TestRepeatedGroups) {
LogicalType::List()));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 1}));
+ EXPECT_THAT(levels, ElementsAre(LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0},
+ LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 1}));
// Arrow schema: list(struct(child: struct(list(bool not null) not null))
non null) not
// null
@@ -2411,16 +2389,16 @@ TEST_F(TestLevels, TestRepeatedGroups) {
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
EXPECT_THAT(
levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
+ ElementsAre(LevelInfo{/*def_level=*/1, /*rep_level=*/1,
/*ancestor_list_def_level*/ 0},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
+ LevelInfo{/*def_level=*/1, /*rep_level=*/1,
/*ancestor_list_def_level*/ 1},
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/2,
/*rep_level=*/1,
+ LevelInfo{/*def_level=*/2, /*rep_level=*/1,
/*ancestor_list_def_level*/ 1}, // optional child
struct
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3,
/*rep_level=*/2,
+ LevelInfo{/*def_level=*/3, /*rep_level=*/2,
/*ancestor_list_def_level*/ 1}, // repeated field
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3,
/*rep_level=*/2,
+ LevelInfo{/*def_level=*/3, /*rep_level=*/2,
/*ancestor_list_def_level*/ 3})); // inner field
// Arrow schema: list(struct(child_list: list(struct(f0: bool f1: bool)))
not null) not
@@ -2438,43 +2416,41 @@ TEST_F(TestLevels, TestRepeatedGroups) {
ParquetType::BOOLEAN)})})},
LogicalType::List())}));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0}, // parent list
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 1}, // parent struct
-
- // Def_level=2 is handled together with def_level=3
- // When decoding. Def_level=2 indicates present but empty
- // list. def_level=3 indicates a present element in the
- // list.
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3,
/*rep_level=*/2,
- /*ancestor_list_def_level*/ 1}, // list field
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/4,
/*rep_level=*/2,
- /*ancestor_list_def_level*/ 3}, // inner struct
field
-
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/5,
/*rep_level=*/2,
- /*ancestor_list_def_level*/ 3})); // f0 bool field
+ EXPECT_THAT(levels,
+ ElementsAre(LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0}, // parent
list
+ LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 1}, // parent
struct
+
+ // Def_level=2 is handled together with def_level=3
+ // When decoding. Def_level=2 indicates present but
empty
+ // list. def_level=3 indicates a present element in
the
+ // list.
+ LevelInfo{/*def_level=*/3, /*rep_level=*/2,
+ /*ancestor_list_def_level*/ 1}, // list
field
+ LevelInfo{/*def_level=*/4, /*rep_level=*/2,
+ /*ancestor_list_def_level*/ 3}, // inner
struct field
+
+ LevelInfo{/*def_level=*/5, /*rep_level=*/2,
+ /*ancestor_list_def_level*/ 3})); // f0
bool field
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/1));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0}, // parent list
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 1}, // parent struct
- // Def_level=2 is handled together with def_level=3
- // When decoding. Def_level=2 indicate present but empty
- // list. def_level=3 indicates a present element in the
- // list.
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3,
/*rep_level=*/2,
- /*ancestor_list_def_level*/ 1}, // list field
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/4,
/*rep_level=*/2,
- /*ancestor_list_def_level*/ 3}, // inner struct
field
-
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/4,
/*rep_level=*/2,
- /*ancestor_list_def_level*/ 3})); // f1 bool field
+ EXPECT_THAT(levels,
+ ElementsAre(LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0}, // parent
list
+ LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 1}, // parent
struct
+ // Def_level=2 is handled together with def_level=3
+ // When decoding. Def_level=2 indicate present but
empty
+ // list. def_level=3 indicates a present element in
the
+ // list.
+ LevelInfo{/*def_level=*/3, /*rep_level=*/2,
+ /*ancestor_list_def_level*/ 1}, // list
field
+ LevelInfo{/*def_level=*/4, /*rep_level=*/2,
+ /*ancestor_list_def_level*/ 3}, // inner
struct field
+
+ LevelInfo{/*def_level=*/4, /*rep_level=*/2,
+ /*ancestor_list_def_level*/ 3})); // f1
bool field
// Arrow schema: list(struct(child_list: list(bool not null)) not null) not
null
// Legacy 2-level encoding (required for backwards compatibility. See
@@ -2488,21 +2464,20 @@ TEST_F(TestLevels, TestRepeatedGroups) {
LogicalType::List())}));
ASSERT_OK_AND_ASSIGN(levels, RootToTreeLeafLevels(*manifest_,
/*column_number=*/0));
- EXPECT_THAT(
- levels,
- ElementsAre(LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 0}, // parent list
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/1,
/*rep_level=*/1,
- /*ancestor_list_def_level*/ 1}, // parent struct
-
- // Def_level=2 is handled together with def_level=3
- // When decoding. Def_level=2 indicate present but empty
- // list. def_level=3 indicates a present element in the
- // list.
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3,
/*rep_level=*/2,
- /*ancestor_list_def_level*/ 1}, // list field
- LevelInfo{/*null_slot_usage=*/1, /*def_level=*/3,
/*rep_level=*/2,
- /*ancestor_list_def_level*/ 3})); // inner bool
+ EXPECT_THAT(levels,
+ ElementsAre(LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 0}, // parent
list
+ LevelInfo{/*def_level=*/1, /*rep_level=*/1,
+ /*ancestor_list_def_level*/ 1}, // parent
struct
+
+ // Def_level=2 is handled together with def_level=3
+ // When decoding. Def_level=2 indicate present but
empty
+ // list. def_level=3 indicates a present element in
the
+ // list.
+ LevelInfo{/*def_level=*/3, /*rep_level=*/2,
+ /*ancestor_list_def_level*/ 1}, // list
field
+ LevelInfo{/*def_level=*/3, /*rep_level=*/2,
+ /*ancestor_list_def_level*/ 3})); //
inner bool
}
TEST_F(TestLevels, ListErrors) {
diff --git a/cpp/src/parquet/level_conversion.cc
b/cpp/src/parquet/level_conversion.cc
index c84007c5cd8..5f0a2dd5795 100644
--- a/cpp/src/parquet/level_conversion.cc
+++ b/cpp/src/parquet/level_conversion.cc
@@ -17,17 +17,13 @@
#include "parquet/level_conversion.h"
-#include <algorithm>
#include <limits>
#include <optional>
-#include "arrow/util/bit_run_reader.h"
-#include "arrow/util/bit_util.h"
#include "arrow/util/cpu_info.h"
-#include "arrow/util/logging.h"
+#include "arrow/util/macros.h"
#include "parquet/exception.h"
-#include "parquet/level_comparison.h"
#if defined(ARROW_HAVE_RUNTIME_BMI2)
# include "parquet/level_conversion_bmi2_internal.h"
#endif
@@ -119,11 +115,6 @@ void DefRepLevelsToListInfo(const int16_t* def_levels,
const int16_t* rep_levels
} else if (valid_bits_writer.has_value()) {
output->values_read = valid_bits_writer->position();
}
- if (output->null_count > 0 && level_info.null_slot_usage > 1) {
- throw ParquetException(
- "Null values with null_slot_usage > 1 not supported."
- "(i.e. FixedSizeLists with null values are not supported)");
- }
}
} // namespace
diff --git a/cpp/src/parquet/level_conversion.h
b/cpp/src/parquet/level_conversion.h
index 31de95be41c..79b9c7518e6 100644
--- a/cpp/src/parquet/level_conversion.h
+++ b/cpp/src/parquet/level_conversion.h
@@ -19,38 +19,27 @@
#include <cstdint>
-#include "arrow/util/endian.h"
#include "parquet/platform.h"
#include "parquet/schema.h"
namespace parquet::internal {
struct PARQUET_EXPORT LevelInfo {
- LevelInfo()
- : null_slot_usage(1), def_level(0), rep_level(0),
repeated_ancestor_def_level(0) {}
- LevelInfo(int32_t null_slots, int32_t definition_level, int32_t
repetition_level,
+ LevelInfo() = default;
+ LevelInfo(int32_t definition_level, int32_t repetition_level,
int32_t repeated_ancestor_definition_level)
- : null_slot_usage(null_slots),
- def_level(static_cast<int16_t>(definition_level)),
+ : def_level(static_cast<int16_t>(definition_level)),
rep_level(static_cast<int16_t>(repetition_level)),
repeated_ancestor_def_level(
static_cast<int16_t>(repeated_ancestor_definition_level)) {}
bool operator==(const LevelInfo& b) const {
- return null_slot_usage == b.null_slot_usage && def_level == b.def_level &&
- rep_level == b.rep_level &&
+ return def_level == b.def_level && rep_level == b.rep_level &&
repeated_ancestor_def_level == b.repeated_ancestor_def_level;
}
bool HasNullableValues() const { return repeated_ancestor_def_level <
def_level; }
- // How many slots an undefined but present (i.e. null) element in
- // parquet consumes when decoding to Arrow.
- // "Slot" is used in the same context as the Arrow specification
- // (i.e. a value holder).
- // This is only ever >1 for descendents of FixedSizeList.
- int32_t null_slot_usage = 1;
-
// The definition level at which the value for the field
// is considered not null (definition levels greater than
// or equal to this value indicate a not-null
@@ -144,11 +133,7 @@ struct PARQUET_EXPORT LevelInfo {
// is not important because all asserts happen directly on
// members.
os << "{def=" << levels.def_level << ", rep=" << levels.rep_level
- << ", repeated_ancestor_def=" << levels.repeated_ancestor_def_level;
- if (levels.null_slot_usage > 1) {
- os << ", null_slot_usage=" << levels.null_slot_usage;
- }
- os << "}";
+ << ", repeated_ancestor_def=" << levels.repeated_ancestor_def_level <<
"}";
return os;
}
};
