lxy-9602 commented on code in PR #483:
URL: https://github.com/apache/paimon-cpp/pull/483#discussion_r4500435552


##########
include/paimon/cache/cache.h:
##########
@@ -44,6 +47,8 @@ class PAIMON_EXPORT CacheKey {
                                                  int32_t length, bool 
is_index);
     static std::shared_ptr<CacheKey> ForKind(const std::string& file_path, 
int64_t position,
                                              int32_t length, CacheKind kind);
+    static std::shared_ptr<CacheKey> ForIndexReader(const std::string& uri,
+                                                    const std::string& 
column_name);

Review Comment:
   It may also require the `index_type`. After all, for a file index, all field 
types’ indexes will be written to the same file.



##########
src/paimon/core/io/file_index_evaluator.h:
##########
@@ -75,7 +76,8 @@ class FileIndexEvaluator {
         const std::shared_ptr<VectorSearch>& vector_search,
         const std::shared_ptr<DataFilePathFactory>& path_factory,
         const std::shared_ptr<DataFileMeta>& file_meta,
-        const std::shared_ptr<FileSystem>& file_system, const 
std::shared_ptr<MemoryPool>& pool);
+        const std::shared_ptr<FileSystem>& file_system, const 
std::shared_ptr<MemoryPool>& pool,
+        const std::shared_ptr<Cache>& cache);

Review Comment:
   Please put the newly added parameter before `pool`, so that `pool` remains 
the last parameter.



##########
src/paimon/core/io/file_index_evaluator.cpp:
##########
@@ -291,14 +292,47 @@ Result<std::shared_ptr<ScoredFileIndexResult>> 
FileIndexEvaluator::EvaluateVecto
     const std::shared_ptr<VectorSearch>& vector_search,
     const std::shared_ptr<DataFilePathFactory>& path_factory,
     const std::shared_ptr<DataFileMeta>& file_meta, const 
std::shared_ptr<FileSystem>& file_system,
-    const std::shared_ptr<MemoryPool>& pool) {
+    const std::shared_ptr<MemoryPool>& pool, const std::shared_ptr<Cache>& 
cache) {
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input_stream,
                            
ExtractIndexInputStream(/*only_use_embedding_index=*/false, path_factory,
                                                    file_meta, file_system));
-    PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<FileIndexReader>> 
readers,
-                           OpenSearchReaders(data_schema, 
vector_search->field_name, options,
-                                             file_meta, input_stream, pool));
-    return ExecuteVectorSearch(readers, vector_search);
+    auto open_reader = [&]() -> Result<std::shared_ptr<FileIndexReader>> {
+        PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<FileIndexReader>> 
readers,
+                               OpenSearchReaders(data_schema, 
vector_search->field_name, options,
+                                                 file_meta, input_stream, 
pool));
+        return GetVectorSearchReader(readers, vector_search->field_name);
+    };
+    std::string uri;
+    if (cache && input_stream) {
+        auto uri_result = input_stream->GetUri();
+        if (uri_result.ok()) {
+            uri = std::move(uri_result).value();
+        }
+    }
+    if (uri.empty()) {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileIndexReader> reader, 
open_reader());
+        return reader->VisitVectorSearch(vector_search);
+    }
+    std::shared_ptr<arrow::Field> field = 
data_schema->GetFieldByName(vector_search->field_name);
+    if (!field) {
+        return Status::Invalid(fmt::format("Search field '{}' does not exist 
in data schema",
+                                           vector_search->field_name));
+    }
+    auto key = CacheKey::ForIndexReader(uri, vector_search->field_name);
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<CacheValue> value,
+        cache->Get(
+            key, [&](const std::shared_ptr<CacheKey>&) -> 
Result<std::shared_ptr<CacheValue>> {
+                PAIMON_ASSIGN_OR_RAISE(int64_t length, input_stream->Length());

Review Comment:
   The index size may be much larger than the available memory, especially in 
DiskANN scenarios. So I’m not sure whether using the index size as the cache 
length is appropriate. Could we add a memory interface for the reader’s 
inherent overhead for Lumina? Or at least leave a TODO for this.



##########
src/paimon/indexer/lumina/lumina_index_searcher.cpp:
##########
@@ -89,30 +90,30 @@ Result<::lumina::api::LuminaSearcher::SearchResult> 
LuminaIndexSearcher::Search(
         if (!vector_search->pre_filter) {
             PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA(
                 ::lumina::api::LuminaSearcher::SearchResult search_result,
-                searcher_with_tag_->SearchWithTag(query, tag_filter, 
search_options, *pool_));
+                searcher_with_tag_->SearchWithTag(query, tag_filter, 
search_options, session_pool));

Review Comment:
   The newly created session pool is taken from the old reader. If two queries 
use different Paimon pools, it seems the memory could be incorrectly mounted. 
I’m wondering whether a cached reader should also carry the session pool and 
use it for the queries.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to