zjw1111 commented on code in PR #504: URL: https://github.com/apache/paimon-cpp/pull/504#discussion_r4509310412
########## src/paimon/core/realtime/index/realtime_index_factory.cpp: ########## @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/realtime/index/realtime_index_factory.h" + +#include "paimon/core/realtime/index/hash/hash_realtime_index.h" +#include "paimon/factories/factory_creator.h" +#include "paimon/realtime/index/realtime_index.h" +#include "paimon/status.h" + +namespace paimon { +const char RealtimeIndexFactory::REALTIME_INDEX_IDENTIFIER_SUFFIX[] = "-realtime"; + +Result<std::shared_ptr<RealtimeIndex>> RealtimeIndexFactory::Get( + const std::string& identifier, ::ArrowSchema* write_schema, + const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool) { + FactoryCreator* creator = FactoryCreator::GetInstance(); + if (!creator) { + return Status::Invalid("factory creator is null pointer"); + } + auto* factory = dynamic_cast<RealtimeIndexFactory*>( + creator->Create(identifier + REALTIME_INDEX_IDENTIFIER_SUFFIX)); + if (!factory) { + return Status::Invalid("unknown real-time index type: ", identifier); + } + return factory->Create(write_schema, options, pool); +} + +// Keep built-in registration with the lookup entry point for static linking. +REGISTER_PAIMON_FACTORY(HashRealtimeIndexFactory); Review Comment: Is this a special design, registered here instead of in the cpp file of the plug-in? ########## src/paimon/core/realtime/index/hash/hash_realtime_index.h: ########## @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include <atomic> +#include <cstdint> +#include <map> +#include <memory> +#include <optional> +#include <string> +#include <vector> + +#include "paimon/common/utils/concurrent_hash_map.h" +#include "paimon/realtime/index/realtime_index.h" +#include "paimon/realtime/index/realtime_index_factory.h" +#include "paimon/realtime/index/realtime_index_reader.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { +class MemoryPool; +enum class FieldType; + +/// Single-writer, multiple-reader hash index: TBB map -> concurrent_vector postings. +/// Atomic lengths publish initialized prefixes without copying. +/// Temporary layout; will be replaced with more lock-friendly data structures. Review Comment: do you need add a `TODO` or `@note` tag? ########## src/paimon/common/io/offset_output_stream.h: ########## @@ -39,6 +39,8 @@ class PAIMON_EXPORT OffsetOutputStream : public OutputStream { Result<int64_t> Write(const char* buffer, int64_t size) override; Result<int64_t> GetPos() const override; + /// Returns the current position of the innermost stream, unwrapping nested OffsetOutputStreams. + Result<int64_t> GetUnderlyingPos() const; Review Comment: move to include ########## src/paimon/core/realtime/arrow_realtime_store.h: ########## @@ -40,16 +41,18 @@ class FileSystem; class IOManager; class MemoryPool; class PredicateFilter; +class RealtimeIndexSet; /// Internal Arrow-backed implementation of the default `RealtimeStore`. class ArrowRealtimeStore final : public RealtimeStoreTwoPhaseSeal { Review Comment: `RealtimeStoreTwoPhaseSeal` rename to like `TwoPhaseSealableRealtimeStore` `SealableRealtimeStore`? and also some var name `two_phase` in other files is also too abstract ########## src/paimon/core/realtime/index/hash/hash_realtime_index.cpp: ########## @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/realtime/index/hash/hash_realtime_index.h" + +#include <algorithm> +#include <atomic> +#include <cstddef> +#include <map> +#include <optional> +#include <string> +#include <utility> +#include <vector> + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "paimon/common/predicate/literal_converter.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/field_type_utils.h" +#include "paimon/core/realtime/realtime_utils.h" +#include "paimon/defs.h" +#include "paimon/macros.h" +#include "tbb/concurrent_vector.h" + +namespace paimon { + +struct HashRealtimeIndex::Posting { + tbb::concurrent_vector<int64_t> offsets; + std::atomic<size_t> published_size{0}; +}; + +HashRealtimeIndex::Reader::Reader(std::shared_ptr<const HashRealtimeIndex> index, + const OffsetRange& offset_range) + : index_(std::move(index)), offset_range_(offset_range) {} + +Result<std::optional<RoaringBitmap64>> HashRealtimeIndex::Reader::VisitEqual( + const Literal& literal) { + return VisitIn({literal}); +} + +Result<std::optional<RoaringBitmap64>> HashRealtimeIndex::Reader::VisitIn( + const std::vector<Literal>& literals) { + RoaringBitmap64 result; + if (offset_range_.Empty()) { + return std::optional<RoaringBitmap64>(std::move(result)); + } + for (const Literal& literal : literals) { + if (literal.IsNull()) { + continue; + } + if (literal.GetType() != index_->field_type_) { + return Status::Invalid("real-time hash index literal type mismatch: expected ", + FieldTypeUtils::FieldTypeToString(index_->field_type_), + ", actual ", + FieldTypeUtils::FieldTypeToString(literal.GetType())); + } + const std::optional<std::shared_ptr<Posting>> posting = + index_->postings_.Find(literal.HashCode()); + if (!posting) { + continue; + } + const Posting& entry = *posting.value(); + const size_t published_size = entry.published_size.load(std::memory_order_acquire); + const auto published_end = entry.offsets.cbegin() + published_size; + auto begin = std::lower_bound(entry.offsets.cbegin(), published_end, offset_range_.begin); + const auto end = std::lower_bound(begin, published_end, offset_range_.end); + // TBB posting segments are not contiguous. + for (; begin != end; ++begin) { + result.Add(*begin); + } + } + return std::optional<RoaringBitmap64>(std::move(result)); +} + +HashRealtimeIndex::HashRealtimeIndex(std::shared_ptr<arrow::Schema> schema, FieldType field_type) + : schema_(std::move(schema)), field_type_(field_type) {} + +Result<std::shared_ptr<RealtimeIndex>> HashRealtimeIndex::Create( + const std::shared_ptr<arrow::Schema>& schema, const std::shared_ptr<MemoryPool>& pool) { + if (!pool) { + return Status::Invalid("real-time hash index requires a memory pool"); + } + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateOffsetField(schema)); + if (schema->num_fields() != 2 || + schema->field(0)->name() != SpecialFields::RealtimeOffset().Name() || + schema->field(1)->name() == SpecialFields::RealtimeOffset().Name()) { + return Status::Invalid( + "real-time hash index requires offset and exactly one indexed field"); + } + PAIMON_ASSIGN_OR_RAISE(FieldType type, FieldTypeUtils::ConvertToFieldType(schema->field(1))); + switch (type) { + case FieldType::BOOLEAN: + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: + case FieldType::BIGINT: + case FieldType::FLOAT: + case FieldType::DOUBLE: + case FieldType::STRING: + case FieldType::BINARY: + case FieldType::DATE: + break; + default: + return Status::NotImplemented("unsupported real-time hash index field type: ", + schema->field(1)->type()->ToString()); + } + return std::shared_ptr<RealtimeIndex>(new HashRealtimeIndex(schema, type)); +} + +Status HashRealtimeIndex::AddBatch(::ArrowArray* batch) { + if (!batch || !batch->release) { + return Status::Invalid("real-time index batch is null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported, + arrow::ImportArray(batch, arrow::struct_(schema_->fields()))); + const std::shared_ptr<arrow::StructArray> data = + checked_pointer_cast<arrow::StructArray>(imported); + const std::shared_ptr<arrow::Int64Array> offsets = checked_pointer_cast<arrow::Int64Array>( + data->GetFieldByName(SpecialFields::RealtimeOffset().Name())); + if (data->null_count() != 0 || offsets->null_count() != 0) { + return Status::Invalid("real-time index requires non-null offsets and rows"); + } + PAIMON_ASSIGN_OR_RAISE(std::vector<Literal> values, LiteralConverter::ConvertLiteralsFromArray( + *data->field(1), /*own_data=*/false)); + + for (int64_t row = 0; row < offsets->length(); ++row) { + const Literal& literal = values[row]; + if (literal.IsNull()) { + continue; + } + const uint64_t hash = literal.HashCode(); + std::optional<std::shared_ptr<Posting>> found = postings_.Find(hash); + std::shared_ptr<Posting> posting; + if (found) { + posting = found.value(); + } else { + posting = std::make_shared<Posting>(); + postings_.Insert(hash, posting); + posting_memory_usage_.fetch_add( + sizeof(Posting) + sizeof(uint64_t) + sizeof(std::shared_ptr<Posting>), + std::memory_order_relaxed); + } + tbb::concurrent_vector<int64_t>& list = posting->offsets; + const size_t old_capacity = list.capacity(); + list.push_back(offsets->Value(row)); + // Publish initialized offsets; concurrent_vector::size() may include unfinished ones. + posting->published_size.store(list.size(), std::memory_order_release); + posting_memory_usage_.fetch_add((list.capacity() - old_capacity) * sizeof(int64_t), + std::memory_order_relaxed); + } + return Status::OK(); +} + +Result<std::shared_ptr<RealtimeIndexReader>> HashRealtimeIndex::CreateReader( + const OffsetRange& offset_range) const { + if (offset_range.begin < 0 || offset_range.begin > offset_range.end) { + return Status::Invalid("real-time index query offset range is invalid"); + } + return std::shared_ptr<RealtimeIndexReader>(new Reader(shared_from_this(), offset_range)); +} + +uint64_t HashRealtimeIndex::GetMemoryUsage() const { + // Estimates payload/capacity; excludes TBB buckets and allocator overhead. + return posting_memory_usage_.load(std::memory_order_relaxed) + sizeof(*this); +} + +const char* HashRealtimeIndexFactory::Identifier() const { + return "hash-realtime"; Review Comment: do you need a more special name, like `tbb-hash`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
