This is an automated email from the ASF dual-hosted git repository.
jiacai2050 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-horaedb.git
The following commit(s) were added to refs/heads/dev by this push:
new 5377dfd5 feat: try load page indexes (#1425)
5377dfd5 is described below
commit 5377dfd5d078ec394d9401703ee5b62bf19fd94d
Author: Jiacai Liu <[email protected]>
AuthorDate: Thu Jan 4 17:04:17 2024 +0800
feat: try load page indexes (#1425)
## Rationale
See #1040
## Detailed Changes
- Try load page indexes
## Test Plan
CI
---
analytic_engine/src/sst/parquet/async_reader.rs | 24 ++++++++++++++++++++++--
1 file changed, 22 insertions(+), 2 deletions(-)
diff --git a/analytic_engine/src/sst/parquet/async_reader.rs
b/analytic_engine/src/sst/parquet/async_reader.rs
index be984796..0b7ffdc2 100644
--- a/analytic_engine/src/sst/parquet/async_reader.rs
+++ b/analytic_engine/src/sst/parquet/async_reader.rs
@@ -37,7 +37,7 @@ use datafusion::{
};
use futures::{Stream, StreamExt};
use generic_error::{BoxError, GenericResult};
-use logger::{debug, error};
+use logger::{debug, error, warn};
use object_store::{ObjectStoreRef, Path};
use parquet::{
arrow::{arrow_reader::RowSelection, ParquetRecordBatchStreamBuilder,
ProjectionMask},
@@ -397,7 +397,27 @@ impl<'a> Reader<'a> {
file_path: self.path.to_string(),
})?;
- // TODO: Support page index until
https://github.com/CeresDB/ceresdb/issues/1040 is fixed.
+ let mut parquet_meta_data = Arc::new(parquet_meta_data);
+ let object_store_reader = parquet_ext::reader::ObjectStoreReader::new(
+ self.store.clone(),
+ self.path.clone(),
+ parquet_meta_data.clone(),
+ );
+
+ if let Ok(meta_data) =
parquet_ext::meta_data::meta_with_page_indexes(object_store_reader)
+ .await
+ .map_err(|e| {
+ // When loading page indexes failed, we just log the error and
continue querying
+ // TODO: Fix this in stream.
https://github.com/apache/incubator-horaedb/issues/1040
+ warn!(
+ "Fail to load page indexes, path:{}, err:{:?}.",
+ self.path, e
+ );
+ e
+ })
+ {
+ parquet_meta_data = meta_data;
+ }
MetaData::try_new(&parquet_meta_data, ignore_sst_filter,
self.store.clone())
.await
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
