qzyu999 commented on code in PR #23656:
URL: https://github.com/apache/datafusion/pull/23656#discussion_r3679380349
##########
datafusion/datasource-parquet/src/sink.rs:
##########
@@ -752,3 +777,278 @@ async fn output_single_parquet_file_parallelized(
.map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))??;
Ok(parquet_meta_data)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::array::{ArrayRef, StringArray};
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::config::TableParquetOptions;
+ use datafusion_datasource::PartitionedFile;
+ use datafusion_datasource::file_groups::FileGroup;
+ use datafusion_datasource::file_sink_config::{FileOutputMode,
FileSinkConfig};
+ use datafusion_datasource::sink::{DataSink, DataSinkExec};
+ use datafusion_datasource::url::ListingTableUrl;
+ use datafusion_execution::config::SessionConfig;
+ use datafusion_execution::object_store::ObjectStoreUrl;
+ use datafusion_execution::runtime_env::RuntimeEnv;
+ use datafusion_expr::dml::InsertOp;
+ use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
+ use object_store::local::LocalFileSystem;
+
+ fn build_test_ctx(store_url: &ObjectStoreUrl) -> Arc<TaskContext> {
+ let tmp_dir = tempfile::TempDir::new().unwrap();
+ let local = Arc::new(
+ LocalFileSystem::new_with_prefix(&tmp_dir)
+ .expect("should create object store"),
+ );
+
+ let session = SessionConfig::default();
+ let runtime = RuntimeEnv::default();
+ runtime
+ .object_store_registry
+ .register_store(store_url.as_ref(), local);
+
+ Arc::new(
+ TaskContext::default()
+ .with_session_config(session)
+ .with_runtime(Arc::new(runtime)),
+ )
+ }
+
+ fn create_test_sink() -> (Arc<ParquetSink>, SchemaRef, ObjectStoreUrl) {
+ let field_a = Field::new("a", DataType::Utf8, false);
+ let field_b = Field::new("b", DataType::Utf8, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let object_store_url = ObjectStoreUrl::local_filesystem();
+
+ let file_sink_config = FileSinkConfig {
+ original_url: String::default(),
+ object_store_url: object_store_url.clone(),
+ file_group:
FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),
+ table_paths:
vec![ListingTableUrl::parse("file:///tmp/test/").unwrap()],
+ output_schema: Arc::clone(&schema),
+ table_partition_cols: vec![],
+ insert_op: InsertOp::Overwrite,
+ keep_partition_by_columns: false,
+ file_extension: "parquet".into(),
+ file_output_mode: FileOutputMode::Automatic,
+ };
Review Comment:
All paths are now derived from `TempDir::path()`:
```rust
let tmp_path = tmp_dir.path().to_owned();
// ...
file_group: FileGroup::new(vec![PartitionedFile::new(
tmp_path.to_string_lossy().to_string(), 1,
)]),
table_paths: vec![ListingTableUrl::parse(
url::Url::from_directory_path(&tmp_path).unwrap().as_str(),
).unwrap()],
```
Added `url = { workspace = true }` to dev-dependencies for
`Url::from_directory_path`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
