lxy-9602 commented on code in PR #483:
URL: https://github.com/apache/paimon-cpp/pull/483#discussion_r4538848682
##########
include/paimon/cache/cache.h:
##########
@@ -88,28 +98,84 @@ class PAIMON_EXPORT Cache {
class PAIMON_EXPORT CacheValue {
public:
- /// Charge the segment's logical size when no separate allocation size is
supplied.
- CacheValue(const MemorySegment& segment, CacheCallback callback);
+ virtual ~CacheValue();
- /// @param memory_usage Retained allocation size in bytes, including
unused capacity.
- /// The charge is at least the segment's logical size.
- CacheValue(const MemorySegment& segment, CacheCallback callback, int64_t
memory_usage);
+ /// Returns the typed payload, retaining its owner; null for a null value
or type mismatch.
+ template <typename T>
+ static std::shared_ptr<T> Get(const std::shared_ptr<CacheValue>& value);
- ~CacheValue();
-
- const MemorySegment& GetSegment() const;
-
- /// Bytes to charge against the cache budget. May exceed
GetSegment().Size().
+ /// Bytes charged against the cache budget.
int64_t GetMemoryUsage() const;
void OnEvict(const std::shared_ptr<CacheKey>& key) const;
- bool operator==(const CacheValue& other) const;
+ virtual bool operator==(const CacheValue& other) const = 0;
+
+ protected:
+ CacheValue(CacheCallback callback, int64_t memory_usage);
private:
- MemorySegment segment_;
CacheCallback callback_;
int64_t memory_usage_;
};
+template <typename T>
+class PAIMON_EXPORT TypedCacheValue : public CacheValue {
+ protected:
+ using CacheValue::CacheValue;
+
+ private:
+ friend class CacheValue;
+ virtual T* GetValuePointer() = 0;
+};
+
+template <typename T>
+std::shared_ptr<T> CacheValue::Get(const std::shared_ptr<CacheValue>& value) {
+ auto* typed_value = dynamic_cast<TypedCacheValue<T>*>(value.get());
+ if (typed_value) {
+ if (T* payload = typed_value->GetValuePointer()) {
+ return std::shared_ptr<T>(value, payload);
+ }
+ }
+ return nullptr;
+}
+
+class PAIMON_EXPORT MemorySegmentCacheValue final : public
TypedCacheValue<const MemorySegment> {
+ public:
+ MemorySegmentCacheValue(const MemorySegment& segment, CacheCallback
callback);
+
+ /// Charges retained allocation size, at least the segment's logical size.
+ MemorySegmentCacheValue(const MemorySegment& segment, CacheCallback
callback,
+ int64_t memory_usage);
+
+ bool operator==(const CacheValue& other) const override;
+
+ private:
+ const MemorySegment* GetValuePointer() override {
+ return &segment_;
+ }
+
+ MemorySegment segment_;
+};
+
+template <typename T>
+class PAIMON_EXPORT ObjectCacheValue final : public TypedCacheValue<T> {
+ public:
+ ObjectCacheValue(std::shared_ptr<T> value, CacheCallback callback, int64_t
memory_usage)
+ : TypedCacheValue<T>(std::move(callback), std::max<int64_t>(1,
memory_usage)),
+ value_(std::move(value)) {}
+
+ bool operator==(const CacheValue& other) const override {
+ const auto* rhs = dynamic_cast<const ObjectCacheValue<T>*>(&other);
+ return rhs && value_ == rhs->value_;
Review Comment:
I wonder should `memory_usage` also be included in the comparison?
##########
src/paimon/common/io/cache/cache.cpp:
##########
@@ -48,11 +39,19 @@ void CacheValue::OnEvict(const std::shared_ptr<CacheKey>&
key) const {
}
}
-bool CacheValue::operator==(const CacheValue& other) const {
- if (this == &other) {
- return true;
- }
- return segment_ == other.segment_;
+MemorySegmentCacheValue::MemorySegmentCacheValue(const MemorySegment& segment,
+ CacheCallback callback)
+ : MemorySegmentCacheValue(segment, std::move(callback), segment.Size()) {}
+
+MemorySegmentCacheValue::MemorySegmentCacheValue(const MemorySegment& segment,
+ CacheCallback callback,
int64_t memory_usage)
+ : TypedCacheValue<const MemorySegment>(std::move(callback),
+ std::max<int64_t>(segment.Size(),
memory_usage)),
+ segment_(segment) {}
+
+bool MemorySegmentCacheValue::operator==(const CacheValue& other) const {
+ const auto* rhs = dynamic_cast<const MemorySegmentCacheValue*>(&other);
+ return rhs && segment_ == rhs->segment_;
Review Comment:
Should we compare `memory_usage` here?
##########
src/paimon/core/io/file_index_evaluator.cpp:
##########
@@ -126,6 +137,30 @@ Result<std::shared_ptr<FileIndexReader>>
FileIndexEvaluator::GetSearchReader(
return Status::Invalid(fmt::format("Data file '{}' has no File Index
for search field '{}'",
file_meta_->file_name, field_name));
}
+ std::vector<std::string> index_types;
+ if (cache_ && search_type == FileIndexReader::SearchType::VECTOR_SEARCH) {
+ index_types = format_reader_->GetIndexTypes(field_name);
+ }
+ // Multiple index types must go through the normal reader selection and
validation.
+ if (index_types.size() == 1 &&
+ index_types.front() == lumina::LuminaFileIndexFactory::IDENTIFIER) {
+ auto key = CacheKey::ForIndexReader(uri_, field_name,
index_types.front());
Review Comment:
I don’t recommend seeing plugin classes in framework code. Could we just
validate that `index_types.size() == 1`? And if the cache misses, we can still
fall back to directly reading.
##########
src/paimon/core/operation/raw_file_split_read_test.cpp:
##########
@@ -60,10 +64,60 @@
#include "paimon/table/source/data_split.h"
#include "paimon/testing/mock/mock_search_file_index.h"
#include "paimon/testing/utils/binary_row_generator.h"
+#include "paimon/testing/utils/counting_cache_test_utils.h"
#include "paimon/testing/utils/read_result_collector.h"
#include "paimon/testing/utils/testharness.h"
namespace paimon::test {
+namespace {
+
+class CacheTrackingFileIndexer : public FileIndexer {
+ public:
+ explicit CacheTrackingFileIndexer(std::shared_ptr<int32_t> open_count)
+ : open_count_(std::move(open_count)),
delegate_(SearchIndexType::VECTOR, nullptr) {}
+
+ std::shared_ptr<FileIndexReader> CreateEmptyReader() const override {
+ return delegate_.CreateEmptyReader();
+ }
+
+ Result<std::shared_ptr<FileIndexReader>> CreateReader(
+ ::ArrowSchema* schema, int64_t start, int64_t length,
+ const std::shared_ptr<InputStream>& input,
+ const std::shared_ptr<MemoryPool>& pool) const override {
+ ++*open_count_;
+ return delegate_.CreateReader(schema, start, length, input, pool);
+ }
+
+ Result<std::shared_ptr<FileIndexWriter>> CreateWriter(
+ ::ArrowSchema* schema, const std::shared_ptr<MemoryPool>& pool) const
override {
+ return delegate_.CreateWriter(schema, pool);
+ }
+
+ private:
+ std::shared_ptr<int32_t> open_count_;
+ MockSearchFileIndexer delegate_;
+};
+
+class CacheTrackingFileIndexFactory : public FileIndexerFactory {
+ public:
+ explicit CacheTrackingFileIndexFactory(std::shared_ptr<int32_t> open_count)
+ : open_count_(std::move(open_count)) {}
+
+ const char* Identifier() const override {
+ return "lumina";
Review Comment:
Would it be possible to consider renaming this identifier? ‘lumina’ here
might be a little confusing. If we relax the ‘lumina’ restriction on the
evaluator side, we could likely choose a clearer name~
##########
src/paimon/core/io/file_index_evaluator.cpp:
##########
@@ -81,18 +83,27 @@ Result<std::unique_ptr<FileIndexEvaluator>>
FileIndexEvaluator::Create(
const std::map<std::string, std::string>& options,
const std::shared_ptr<DataFilePathFactory>& path_factory,
const std::shared_ptr<DataFileMeta>& file_meta, const
std::shared_ptr<FileSystem>& file_system,
- const std::shared_ptr<MemoryPool>& pool) {
+ const std::shared_ptr<Cache>& cache, const std::shared_ptr<MemoryPool>&
pool) {
+ std::shared_ptr<InputStream> input_stream;
std::unique_ptr<FileIndexFormat::Reader> format_reader;
if (file_meta->row_count >= 0 && file_meta->row_count <=
RoaringBitmap32::MAX_VALUE) {
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input_stream,
+ PAIMON_ASSIGN_OR_RAISE(input_stream,
ExtractIndexInputStream(path_factory,
file_meta, file_system));
if (input_stream) {
PAIMON_ASSIGN_OR_RAISE(format_reader,
FileIndexFormat::CreateReader(input_stream,
pool, options));
}
}
- return std::unique_ptr<FileIndexEvaluator>(
+ auto evaluator = std::unique_ptr<FileIndexEvaluator>(
new FileIndexEvaluator(data_schema, file_meta,
std::move(format_reader)));
+ if (cache && input_stream) {
+ auto uri = input_stream->GetUri();
+ if (uri.ok() && !uri.value().empty()) {
+ evaluator->cache_ = cache;
+ evaluator->uri_ = std::move(uri).value();
+ }
+ }
Review Comment:
Could we move `cache` and `uri` into the constructor?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]