mkleen commented on code in PR #22613:
URL: https://github.com/apache/datafusion/pull/22613#discussion_r3323078885
##########
datafusion/execution/src/cache/cache_manager.rs:
##########
@@ -15,31 +15,61 @@
// specific language governing permissions and limitations
// under the License.
-use crate::cache::CacheAccessor;
-use crate::cache::DefaultListFilesCache;
-use crate::cache::file_statistics_cache::{
- DEFAULT_FILE_STATISTICS_MEMORY_LIMIT, DefaultFileStatisticsCache,
- DefaultFilesMetadataCache,
-};
-use crate::cache::list_files_cache::ListFilesEntry;
-use crate::cache::list_files_cache::TableScopedPath;
-use datafusion_common::TableReference;
+use crate::cache::cache::DefaultCache;
+use crate::cache::{Cache, Value};
use datafusion_common::heap_size::{DFHeapSize, DFHeapSizeCtx};
use datafusion_common::stats::Precision;
+use datafusion_common::{HashMap, TableReference};
use datafusion_common::{Result, Statistics};
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use object_store::ObjectMeta;
use object_store::path::Path;
use std::any::Any;
-use std::collections::HashMap;
-use std::fmt::{Debug, Formatter};
+use std::fmt::{Debug, Display, Formatter};
use std::ops::Deref;
use std::sync::Arc;
use std::time::Duration;
-pub use super::list_files_cache::{
- DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT, DEFAULT_LIST_FILES_CACHE_TTL,
-};
+/// Calculates the number of bytes an [`ObjectMeta`] occupies in the heap.
+pub fn meta_heap_bytes(object_meta: &ObjectMeta) -> usize {
Review Comment:
This is a left-over from list-files-cache which didn't use the `DFHeapsize`
trait for heap-size accounting because it did not exist at the time.
`DFHeapsize` cannot be implemented here for `ObjectMeta` because it would
become an orphan. To remove it and use `DFHeapsize` this would require one of
the following changes:
- Add a dependency to `object_store` to the package commons where
`DFHeapsize` is currently located and implement it for `ObjectMeta`. The
downside is that this would spread the `object_store` further around.
- Move the `DFHeapsize` trait into execution package, since it's nowhere
else used yet.
- Create a new crate/package for the heap-size estimation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
