comphead commented on code in PR #8531:
URL: https://github.com/apache/arrow-datafusion/pull/8531#discussion_r1425998143
##########
datafusion/core/src/datasource/physical_plan/parquet/mod.rs:
##########
@@ -1938,6 +1938,97 @@ mod tests {
Ok(schema)
}
+ #[tokio::test]
+ async fn write_table_results() -> Result<()> {
+ // create partitioned input file and context
+ let tmp_dir = TempDir::new()?;
+ // let mut ctx = create_ctx(&tmp_dir, 4).await?;
+ let ctx = SessionContext::new_with_config(
+ SessionConfig::new().with_target_partitions(8),
+ );
+ let schema = populate_csv_partitions(&tmp_dir, 4, ".csv")?;
+ // register csv file with the execution context
+ ctx.register_csv(
+ "test",
+ tmp_dir.path().to_str().unwrap(),
+ CsvReadOptions::new().schema(&schema),
+ )
+ .await?;
+
+ // register a local file system object store for /tmp directory
+ let local = Arc::new(LocalFileSystem::new_with_prefix(&tmp_dir)?);
+ let local_url = Url::parse("file://local").unwrap();
+ ctx.runtime_env().register_object_store(&local_url, local);
+
+ // Configure listing options
+ let file_format =
ParquetFormat::default().with_enable_pruning(Some(true));
+ let listing_options = ListingOptions::new(Arc::new(file_format))
+ .with_file_extension(FileType::PARQUET.get_ext());
+
+ // execute a simple query and write the results to parquet
+ let out_dir = tmp_dir.as_ref().to_str().unwrap().to_string() + "/out";
+ std::fs::create_dir(&out_dir).unwrap();
+ let df = ctx.sql("SELECT c1, c2 FROM test").await?;
+ let schema: Schema = df.schema().into();
+ // Register a listing table - this will use all files in the directory
as data sources
+ // for the query
+ ctx.register_listing_table(
+ "my_table",
+ &out_dir,
+ listing_options,
+ Some(Arc::new(schema)),
+ None,
+ )
+ .await
+ .unwrap();
+ df.write_table("my_table", DataFrameWriteOptions::new())
+ .await?;
+
+ // create a new context and verify that the results were saved to a
partitioned parquet file
+ let ctx = SessionContext::new();
+
+ // get write_id
+ let mut paths = fs::read_dir(&out_dir).unwrap();
+ let path = paths.next();
+ let name = path
+ .unwrap()?
+ .path()
+ .file_name()
+ .expect("Should be a file name")
+ .to_str()
+ .expect("Should be a str")
+ .to_owned();
+ println!("{name}");
Review Comment:
lets remove it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
