alamb commented on code in PR #9372:
URL: https://github.com/apache/arrow-datafusion/pull/9372#discussion_r1505202349
##########
datafusion-examples/examples/catalog.rs:
##########
@@ -36,94 +33,102 @@ use datafusion::{
prelude::SessionContext,
};
use std::sync::RwLock;
-use std::{
- any::Any,
- collections::HashMap,
- path::{Path, PathBuf},
- sync::Arc,
-};
+use std::{any::Any, collections::HashMap, path::Path, sync::Arc};
+use std::{fs::File, io::Write};
+use tempfile::TempDir;
#[tokio::main]
async fn main() -> Result<()> {
- let repo_dir = std::fs::canonicalize(
- PathBuf::from(env!("CARGO_MANIFEST_DIR"))
- // parent dir of datafusion-examples = repo root
- .join(".."),
- )
- .unwrap();
+ env_logger::builder()
Review Comment:
👍
##########
datafusion-examples/examples/catalog.rs:
##########
@@ -36,94 +33,102 @@ use datafusion::{
prelude::SessionContext,
};
use std::sync::RwLock;
-use std::{
- any::Any,
- collections::HashMap,
- path::{Path, PathBuf},
- sync::Arc,
-};
+use std::{any::Any, collections::HashMap, path::Path, sync::Arc};
+use std::{fs::File, io::Write};
+use tempfile::TempDir;
#[tokio::main]
async fn main() -> Result<()> {
- let repo_dir = std::fs::canonicalize(
- PathBuf::from(env!("CARGO_MANIFEST_DIR"))
- // parent dir of datafusion-examples = repo root
- .join(".."),
- )
- .unwrap();
+ env_logger::builder()
+ .filter_level(log::LevelFilter::Info)
+ .init();
+
+ // Prepare test directories containing multiple files
+ let dir_a = prepare_example_data()?;
+ let dir_b = prepare_example_data()?;
+
let mut ctx = SessionContext::new();
let state = ctx.state();
let catlist = Arc::new(CustomCatalogProviderList::new());
+
// use our custom catalog list for context. each context has a single
catalog list.
// context will by default have [`MemoryCatalogProviderList`]
ctx.register_catalog_list(catlist.clone());
// initialize our catalog and schemas
let catalog = DirCatalog::new();
- let parquet_schema = DirSchema::create(
+ let schema_a = DirSchema::create(
&state,
DirSchemaOpts {
- format: Arc::new(ParquetFormat::default()),
- dir: &repo_dir.join("parquet-testing").join("data"),
- ext: "parquet",
+ format: Arc::new(CsvFormat::default()),
+ dir: dir_a.path(),
+ ext: "csv",
},
)
.await?;
- let csv_schema = DirSchema::create(
+ let schema_b = DirSchema::create(
&state,
DirSchemaOpts {
format: Arc::new(CsvFormat::default()),
- dir: &repo_dir.join("testing").join("data").join("csv"),
+ dir: dir_b.path(),
ext: "csv",
},
)
.await?;
+
// register schemas into catalog
- catalog.register_schema("parquet", parquet_schema.clone())?;
- catalog.register_schema("csv", csv_schema.clone())?;
+ catalog.register_schema("schema_a", schema_a.clone())?;
+ catalog.register_schema("schema_b", schema_b.clone())?;
+
// register our catalog in the context
ctx.register_catalog("dircat", Arc::new(catalog));
{
// catalog was passed down into our custom catalog list since we
overide the ctx's default
let catalogs = catlist.catalogs.read().unwrap();
assert!(catalogs.contains_key("dircat"));
};
- // take the first 5 (arbitrary amount) keys from our schema's hashmap.
+
+ // take the first 3 (arbitrary amount) keys from our schema's hashmap.
Review Comment:
Thie code still seems to take 5 (not 3)
```suggestion
// take the first 5 (arbitrary amount) keys from our schema's hashmap.
```
##########
datafusion-examples/Cargo.toml:
##########
@@ -41,10 +41,6 @@ path = "examples/flight/flight_server.rs"
name = "flight_client"
path = "examples/flight/flight_client.rs"
-[[example]]
Review Comment:
This is removed because the code is now in `examples/catalog.rs` and thus
gets an implicit target
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]