mapleFU commented on code in PR #37400: URL: https://github.com/apache/arrow/pull/37400#discussion_r1527253863
########## cpp/src/parquet/bloom_filter_builder.h: ########## @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/io/type_fwd.h" +#include "parquet/types.h" + +namespace parquet { + +class BloomFilter; +class SchemaDescriptor; +struct BloomFilterOptions; +struct BloomFilterLocation; + +/// \brief Interface for collecting bloom filter of a parquet file. +class PARQUET_EXPORT BloomFilterBuilder { + public: + /// \brief API convenience to create a BloomFilterBuilder. + static std::unique_ptr<BloomFilterBuilder> Make(const SchemaDescriptor* schema, + const WriterProperties& properties); + + /// Append a new row group to host all incoming bloom filters. + /// + /// This method must be called before Finish. + virtual void AppendRowGroup() = 0; + + /// \brief Get the BloomFilter from column ordinal. + /// + /// \param column_ordinal Column ordinal in schema, which is only for leaf columns. + /// + /// \return BloomFilter for the column and its memory ownership belongs to the + /// BloomFilterBuilder. + virtual BloomFilter* GetOrCreateBloomFilter(int32_t column_ordinal) = 0; Review Comment: I've update the doc, would you mind take a look again? ########## cpp/src/parquet/bloom_filter_builder.h: ########## @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/io/type_fwd.h" +#include "parquet/types.h" + +namespace parquet { + +class BloomFilter; +class SchemaDescriptor; +struct BloomFilterOptions; +struct BloomFilterLocation; + +/// \brief Interface for collecting bloom filter of a parquet file. +class PARQUET_EXPORT BloomFilterBuilder { + public: + /// \brief API convenience to create a BloomFilterBuilder. + static std::unique_ptr<BloomFilterBuilder> Make(const SchemaDescriptor* schema, + const WriterProperties& properties); + + /// Append a new row group to host all incoming bloom filters. + /// + /// This method must be called before Finish. Review Comment: I've updated the comment here. PTAL again ########## cpp/src/parquet/bloom_filter_builder.h: ########## @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/io/type_fwd.h" +#include "parquet/types.h" + +namespace parquet { + +class BloomFilter; +class SchemaDescriptor; +struct BloomFilterOptions; +struct BloomFilterLocation; + +/// \brief Interface for collecting bloom filter of a parquet file. +class PARQUET_EXPORT BloomFilterBuilder { + public: + /// \brief API convenience to create a BloomFilterBuilder. Review Comment: Changing it to `/// \brief API to create a BloomFilterBuilder.` ########## cpp/src/parquet/bloom_filter_reader_writer_test.cc: ########## @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <arrow/testing/gtest_util.h> +#include <gtest/gtest.h> + +#include "parquet/bloom_filter.h" +#include "parquet/bloom_filter_builder.h" +#include "parquet/bloom_filter_reader.h" +#include "parquet/file_reader.h" +#include "parquet/test_util.h" + +namespace parquet::test { + +TEST(BloomFilterReader, ReadBloomFilter) { + std::string dir_string(parquet::test::get_data_dir()); + std::string path = dir_string + "/data_index_bloom_encoding_stats.parquet"; + auto reader = ParquetFileReader::OpenFile(path, false); + auto file_metadata = reader->metadata(); + EXPECT_FALSE(file_metadata->is_encryption_algorithm_set()); + auto& bloom_filter_reader = reader->GetBloomFilterReader(); + auto row_group_0 = bloom_filter_reader.RowGroup(0); + ASSERT_NE(nullptr, row_group_0); + EXPECT_THROW(bloom_filter_reader.RowGroup(1), ParquetException); + auto bloom_filter = row_group_0->GetColumnBloomFilter(0); + ASSERT_NE(nullptr, bloom_filter); + EXPECT_THROW(row_group_0->GetColumnBloomFilter(1), ParquetException); + + // assert exists + { + std::string_view sv = "Hello"; + ByteArray ba{sv}; + EXPECT_TRUE(bloom_filter->FindHash(bloom_filter->Hash(&ba))); + } + + // no exists + { + std::string_view sv = "NOT_EXISTS"; + ByteArray ba{sv}; + EXPECT_FALSE(bloom_filter->FindHash(bloom_filter->Hash(&ba))); + } +} + +TEST(BloomFilterReader, FileNotHaveBloomFilter) { + // Can still get a BloomFilterReader and a RowGroupBloomFilter + // reader, but cannot get a non-null BloomFilter. + std::string dir_string(parquet::test::get_data_dir()); + std::string path = dir_string + "/alltypes_plain.parquet"; + auto reader = ParquetFileReader::OpenFile(path, false); + auto file_metadata = reader->metadata(); + EXPECT_FALSE(file_metadata->is_encryption_algorithm_set()); + auto& bloom_filter_reader = reader->GetBloomFilterReader(); + auto row_group_0 = bloom_filter_reader.RowGroup(0); + ASSERT_NE(nullptr, row_group_0); + EXPECT_THROW(bloom_filter_reader.RowGroup(1), ParquetException); + auto bloom_filter = row_group_0->GetColumnBloomFilter(0); + ASSERT_EQ(nullptr, bloom_filter); +} + +// <c1:BYTE_ARRAY, c2:BYTE_ARRAY>, c1 has bloom filter. +TEST(BloomFilterBuilderTest, BasicRoundTrip) { + SchemaDescriptor schema; + schema::NodePtr root = schema::GroupNode::Make( + "schema", Repetition::REPEATED, {schema::ByteArray("c1"), schema::ByteArray("c2")}); + schema.Init(root); + WriterProperties::Builder properties_builder; + BloomFilterOptions bloom_filter_options; + bloom_filter_options.ndv = 100; + properties_builder.enable_bloom_filter_options(bloom_filter_options, "c1"); + auto writer_properties = properties_builder.build(); + auto builder = BloomFilterBuilder::Make(&schema, *writer_properties); + builder->AppendRowGroup(); + auto bloom_filter = builder->GetOrCreateBloomFilter(0); + ASSERT_NE(nullptr, bloom_filter); + ASSERT_EQ(bloom_filter->GetBitsetSize(), + BlockSplitBloomFilter::OptimalNumOfBytes(bloom_filter_options.ndv, + bloom_filter_options.fpp)); + std::vector<uint64_t> insert_hashes = {100, 200}; + for (uint64_t hash : insert_hashes) { + bloom_filter->InsertHash(hash); + } + builder->Finish(); + auto sink = CreateOutputStream(); + BloomFilterLocation location; + builder->WriteTo(sink.get(), &location); + EXPECT_EQ(1, location.bloom_filter_location.size()); + EXPECT_EQ(2, location.bloom_filter_location[0].size()); + EXPECT_TRUE(location.bloom_filter_location[0][0].has_value()); + EXPECT_FALSE(location.bloom_filter_location[0][1].has_value()); + + int64_t bloom_filter_offset = location.bloom_filter_location[0][0]->offset; + int32_t bloom_filter_length = location.bloom_filter_location[0][0]->length; + + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + ReaderProperties reader_properties; + ::arrow::io::BufferReader reader( + ::arrow::SliceBuffer(buffer, bloom_filter_offset, bloom_filter_length)); + auto filter = parquet::BlockSplitBloomFilter::Deserialize(reader_properties, &reader); + for (uint64_t hash : insert_hashes) { + EXPECT_TRUE(bloom_filter->FindHash(hash)); + } + EXPECT_FALSE(filter.FindHash(300)); +} + Review Comment: You're right, will add one -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
