wiedld commented on code in PR #7379:
URL: https://github.com/apache/arrow-datafusion/pull/7379#discussion_r1302270314
##########
datafusion/core/src/physical_plan/sorts/merge.rs:
##########
@@ -15,95 +15,20 @@
// specific language governing permissions and limitations
// under the License.
-//! Merge that deals with an arbitrary size of streaming inputs.
-//! This is an order-preserving merge.
-
use crate::physical_plan::metrics::BaselineMetrics;
-use crate::physical_plan::sorts::builder::BatchBuilder;
+use crate::physical_plan::sorts::builder::{SortOrder, SortOrderBuilder};
use crate::physical_plan::sorts::cursor::Cursor;
-use crate::physical_plan::sorts::stream::{
- FieldCursorStream, PartitionedStream, RowCursorStream,
-};
-use crate::physical_plan::{
- PhysicalSortExpr, RecordBatchStream, SendableRecordBatchStream,
-};
-use arrow::datatypes::{DataType, SchemaRef};
+use crate::physical_plan::sorts::stream::CursorStream;
use arrow::record_batch::RecordBatch;
-use arrow_array::*;
use datafusion_common::Result;
use datafusion_execution::memory_pool::MemoryReservation;
use futures::Stream;
use std::pin::Pin;
use std::task::{ready, Context, Poll};
-macro_rules! primitive_merge_helper {
- ($t:ty, $($v:ident),+) => {
- merge_helper!(PrimitiveArray<$t>, $($v),+)
- };
-}
-
-macro_rules! merge_helper {
- ($t:ty, $sort:ident, $streams:ident, $schema:ident,
$tracking_metrics:ident, $batch_size:ident, $fetch:ident, $reservation:ident)
=> {{
- let streams = FieldCursorStream::<$t>::new($sort, $streams);
- return Ok(Box::pin(SortPreservingMergeStream::new(
- Box::new(streams),
- $schema,
- $tracking_metrics,
- $batch_size,
- $fetch,
- $reservation,
- )));
- }};
-}
-
-/// Perform a streaming merge of [`SendableRecordBatchStream`] based on
provided sort expressions
-/// while preserving order.
-pub fn streaming_merge(
Review Comment:
Since we added a layer between `streaming_merge()` and
`SortPreservingMergeStream` (a.k.a. the cascading stream layer) => therefore,
decided to move this function to it's own mod.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
