alamb commented on code in PR #19319:
URL: https://github.com/apache/datafusion/pull/19319#discussion_r2662778485
##########
datafusion-examples/examples/data_io/parquet_encrypted.rs:
##########
@@ -26,40 +26,25 @@ use datafusion::dataframe::{DataFrame,
DataFrameWriteOptions};
use datafusion::logical_expr::{col, lit};
use datafusion::parquet::encryption::decrypt::FileDecryptionProperties;
use datafusion::parquet::encryption::encrypt::FileEncryptionProperties;
-use datafusion::prelude::CsvReadOptions;
use datafusion::prelude::{ParquetReadOptions, SessionContext};
+use datafusion_examples::utils::write_csv_to_parquet;
use tempfile::TempDir;
-use tokio::fs::create_dir_all;
/// Read and write encrypted Parquet files using DataFusion
pub async fn parquet_encrypted() -> datafusion::common::Result<()> {
// The SessionContext is the main high level API for interacting with
DataFusion
let ctx = SessionContext::new();
- // Load CSV into an in-memory DataFrame, then materialize it to Parquet.
- // This replaces a static parquet fixture and makes the example
self-contained
- // without requiring DataFusion test files.
- let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+ // Convert the CSV input into a temporary Parquet directory for querying
+ let csv_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("data")
.join("csv")
.join("cars.csv");
- let csv_df = ctx
- .read_csv(path.to_str().unwrap(), CsvReadOptions::default())
- .await?;
- let tmp_source = TempDir::new()?;
- let out_dir = tmp_source.path().join("parquet_source");
- create_dir_all(&out_dir).await?;
- csv_df
- .write_parquet(
- out_dir.to_str().unwrap(),
- DataFrameWriteOptions::default(),
- None,
- )
- .await?;
+ let parquet_temp = write_csv_to_parquet(&ctx, &csv_path).await?;
Review Comment:
this is great
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
