yangxk1 commented on code in PR #861:
URL: https://github.com/apache/incubator-graphar/pull/861#discussion_r2870268637
##########
cpp/src/graphar/arrow/chunk_reader.cc:
##########
@@ -352,17 +354,22 @@ Status VertexPropertyArrowChunkReader::next_chunk() {
vertex_info_->GetType(), " chunk num ", chunk_num_);
}
seek_id_ = chunk_index_ * vertex_info_->GetChunkSize();
- chunk_table_.reset();
+ auto* cached = chunk_cache_.Get(chunk_index_);
+ chunk_table_ = cached ? *cached : nullptr;
return Status::OK();
}
void VertexPropertyArrowChunkReader::Filter(util::Filter filter) {
filter_options_.filter = filter;
+ chunk_table_ = nullptr;
+ chunk_cache_.Clear();
}
void VertexPropertyArrowChunkReader::Select(util::ColumnNames column_names) {
filter_options_.columns = column_names;
+ chunk_table_ = nullptr;
Review Comment:
Should it be initialized in this `xxxSelect` function?
##########
cpp/src/graphar/arrow/chunk_reader.cc:
##########
@@ -352,17 +354,22 @@ Status VertexPropertyArrowChunkReader::next_chunk() {
vertex_info_->GetType(), " chunk num ", chunk_num_);
}
seek_id_ = chunk_index_ * vertex_info_->GetChunkSize();
- chunk_table_.reset();
+ auto* cached = chunk_cache_.Get(chunk_index_);
+ chunk_table_ = cached ? *cached : nullptr;
return Status::OK();
}
void VertexPropertyArrowChunkReader::Filter(util::Filter filter) {
filter_options_.filter = filter;
+ chunk_table_ = nullptr;
Review Comment:
Should it be initialized in this `xxxFilter` function?
##########
cpp/src/graphar/arrow/chunk_reader.h:
##########
@@ -267,6 +268,7 @@ class VertexPropertyArrowChunkReader {
IdType vertex_num_;
std::shared_ptr<arrow::Schema> schema_;
std::shared_ptr<arrow::Table> chunk_table_;
+ LRUCache<IdType, std::shared_ptr<arrow::Table>> chunk_cache_{4};
Review Comment:
chunk_cache **size** can judge based on memory or let the user control it
will be better?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
