baojinri commented on code in PR #1586:
URL: https://github.com/apache/horaedb/pull/1586#discussion_r1828926924
##########
horaedb/metric_engine/src/storage.rs:
##########
@@ -206,7 +226,39 @@ impl TimeMergeStorage for CloudObjectStorage {
}
async fn scan(&self, req: ScanRequest) ->
Result<SendableRecordBatchStream> {
- todo!()
+ let ssts = self.manifest.find_ssts(&req.range).await;
+ // we won't use url for selecting object_store.
+ let dummy_url = ObjectStoreUrl::parse("empty://").unwrap();
+ // TODO: we could group ssts based on time range.
+ // TODO: fetch using multiple threads since read from parquet will
incur CPU
+ // when convert between arrow and parquet.
+ let file_groups = ssts
+ .iter()
+ .map(|f| PartitionedFile::new(self.build_file_path(f.id),
f.meta.size as u64))
+ .collect::<Vec<_>>();
+ let scan_config = FileScanConfig::new(dummy_url, self.schema().clone())
+ .with_file_group(file_groups)
+ .with_projection(req.projections);
+
+ let mut builder =
ParquetExec::builder(scan_config).with_parquet_file_reader_factory(
+ Arc::new(DefaultParquetFileReaderFactory::new(self.store.clone())),
+ );
+ if let Some(expr) = conjunction(req.predicate) {
+ let filters = create_physical_expr(&expr, &self.df_schema,
&ExecutionProps::new())
+ .context("create pyhsical expr")?;
+ builder = builder.with_predicate(filters);
+ }
+
+ let parquet_exec = builder.build();
+ let sort_exprs = self.build_sort_exprs()?;
+ let physical_plan = Arc::new(SortExec::new(sort_exprs,
Arc::new(parquet_exec)));
+
+ let ctx = SessionContext::default();
+ // TODO: dedup record batch based on primary keys and sequence number.
+ let res =
+ execute_stream(physical_plan, ctx.task_ctx()).context("execute
sort physical plan")?;
Review Comment:
execute scan physical plan ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
