kosiew commented on code in PR #23975: URL: https://github.com/apache/datafusion/pull/23975#discussion_r3765228929
########## benchmarks/src/statistics.rs: ########## @@ -0,0 +1,760 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Reports planning statistics alongside runtime metrics for benchmark queries. + +use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::{Arc, LazyLock}; + +use clap::Args; +use datafusion::error::{DataFusionError, Result}; +use datafusion::physical_plan::metrics::MetricValue; +use datafusion::physical_plan::operator_statistics::StatisticsRegistry; +use datafusion::physical_plan::{ExecutionPlan, collect}; +use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext}; +use datafusion::sql::parser::{DFParserBuilder, Statement}; +use datafusion::sql::sqlparser::dialect::dialect_from_str; +use datafusion_common::config::{Dialect, SqlParserOptions}; +use datafusion_common::config_err; +use datafusion_common::stats::Precision; +use regex::Regex; +use serde::{Deserialize, Serialize}; + +/// Generate reports that compare planning statistics with runtime metrics. +#[derive(Debug, Args)] +#[command(verbatim_doc_comment)] +pub struct RunOpt { + /// Query filename stem. If not specified, runs every `.sql` file. + #[arg(short, long)] + query: Option<String>, + + /// Branch whose results should be compared. Defaults to the previous run on this branch. + #[arg(long)] + compare: Option<String>, + + /// Path to Parquet data. Top-level files and directories are registered as tables. + #[arg(required = true, short = 'p', long)] + path: PathBuf, + + /// Path to a SQL file or directory of SQL query files. + #[arg(required = true, short = 'Q', long = "query_path")] + query_path: PathBuf, +} + +impl RunOpt { + pub async fn run(self) -> Result<()> { + let mut config = SessionConfig::from_env()?.with_collect_statistics(true); + config.options_mut().optimizer.prefer_hash_join = true; + let ctx = SessionContext::new_with_config(config); + let sql_parser_options = ctx.state().config_options().sql_parser.clone(); + register_parquet_files(&ctx, &self.path).await?; + + let branch = current_branch_name(); + let result_path = self.report_path(&branch); + let comparison_branch = self + .compare + .as_deref() + .map_or(branch.as_str(), |branch| branch); + let comparison_path = self.report_path(comparison_branch); + let previous = load_comparison_report(&comparison_path)?; + backup_previous_report(&result_path)?; + let comparison_description = self.compare.as_ref().map_or_else( + || format!("previous run on branch '{branch}'"), + |branch| format!("branch '{branch}'"), + ); + + let mut reports = vec![]; + for query_path in query_files(&self.query_path, self.query.as_deref())? { + let query = query_path + .file_stem() + .expect("query file has a filename") + .to_string_lossy() + .to_string(); + let sql = fs::read_to_string(query_path)?; + let statements = match sql_statements(&sql, &sql_parser_options) { + Ok(statements) => statements, + Err(error) => { + let report = QueryReport { + query: query.clone(), + statement: 1, + operators: vec![], + success: false, + error: Some(error.to_string()), + }; + print_query_report(&report, previous.as_deref()); + reports.push(report); + store_report(&result_path, &reports)?; + continue; + } + }; + for (statement, sql) in statements.into_iter().enumerate() { + let statement = statement + 1; + let report = match self.report_statement(&ctx, sql).await { + Ok(operators) => QueryReport { + query: query.clone(), + statement, + operators, + success: true, + error: None, + }, + Err(error) => QueryReport { + query: query.clone(), + statement, + operators: vec![], + success: false, + error: Some(error.to_string()), + }, + }; + print_query_report(&report, previous.as_deref()); + reports.push(report); + store_report(&result_path, &reports)?; + } + } + print_q_error_summary( + &reports, + previous.as_deref(), + &branch, + &comparison_description, + ); + Ok(()) + } + + fn report_path(&self, branch: &str) -> PathBuf { + let report_name = self.query.as_ref().map_or_else( + || "statistics.json".to_string(), + |query| format!("statistics-{query}.json"), + ); + PathBuf::from("target/dfbench/statistics") + .join(normalize_branch_name(branch)) + .join(report_name) + } + + async fn report_statement( + &self, + ctx: &SessionContext, + statement: Statement, + ) -> Result<Vec<OperatorReport>> { + let state = ctx.state(); + let logical_plan = state.statement_to_plan(statement).await?; Review Comment: I think this introduces a regression for multi-statement suites that contain session-changing statements. `report_statement` calls `SessionState::statement_to_plan`, optimizes the result, and then directly creates and collects the physical plan. That bypasses the `SessionContext` execution path that applies session-affecting logical plans such as DDL. For example, `CREATE TABLE x AS VALUES (1); SELECT * FROM x;` currently reports the first statement as `Unsupported logical plan: CreateMemoryTable`, and the second statement then cannot find `x`. The earlier `ctx.sql` path applied the session change before continuing. Could we route session-changing statements through the context execution path while preserving the metric-collection path for query plans? Alternatively, if this command is intended to support query statements only, we should reject non-query statements explicitly and document that restriction. It would also be good to add a regression test with a session-changing statement followed by a query. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
