This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d47a40e ARROW-5680: [Rust] [DataFusion] GROUP BY sql tests are now
deterministic
d47a40e is described below
commit d47a40e88292840b9c73bec59af1828b273445e8
Author: Andy Grove <[email protected]>
AuthorDate: Sat Oct 12 10:37:11 2019 -0600
ARROW-5680: [Rust] [DataFusion] GROUP BY sql tests are now deterministic
DataFusion doesn't support `ORDER BY` yet, so I modified the aggregate
tests to collect `Vec<String>` and sort it before comparing against expected
results, making these tests deterministic.
The PR is a little noisy because I removed the final `\n` from the expected
results in most of the tests due to the new approach of collecting
`Vec<String>` and then performing a `.join("\n")` to create the string.
Closes #5622 from andygrove/ARROW-5680 and squashes the following commits:
a3083c6cb <Andy Grove> GROUP BY sql tests are now deterministic
Authored-by: Andy Grove <[email protected]>
Signed-off-by: Andy Grove <[email protected]>
---
rust/datafusion/tests/sql.rs | 141 +++++++++++++++----------------------------
1 file changed, 47 insertions(+), 94 deletions(-)
diff --git a/rust/datafusion/tests/sql.rs b/rust/datafusion/tests/sql.rs
index bb60603..b9741ce 100644
--- a/rust/datafusion/tests/sql.rs
+++ b/rust/datafusion/tests/sql.rs
@@ -87,8 +87,10 @@ fn parquet_query() {
let mut ctx = ExecutionContext::new();
register_alltypes_parquet(&mut ctx);
let sql = "SELECT id, string_col FROM alltypes_plain";
- let actual = execute(&mut ctx, sql);
- let expected =
"4\t\"0\"\n5\t\"1\"\n6\t\"0\"\n7\t\"1\"\n2\t\"0\"\n3\t\"1\"\n0\t\"0\"\n1\t\"1\"\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected =
+
"4\t\"0\"\n5\t\"1\"\n6\t\"0\"\n7\t\"1\"\n2\t\"0\"\n3\t\"1\"\n0\t\"0\"\n1\t\"1\""
+ .to_string();
assert_eq!(expected, actual);
}
@@ -114,8 +116,8 @@ fn csv_count_star() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
let sql = "SELECT COUNT(*), COUNT(1), COUNT(c1) FROM aggregate_test_100";
- let actual = execute(&mut ctx, sql);
- let expected = "100\t100\t100\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected = "100\t100\t100".to_string();
assert_eq!(expected, actual);
}
@@ -124,8 +126,8 @@ fn csv_query_with_predicate() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
let sql = "SELECT c1, c12 FROM aggregate_test_100 WHERE c12 > 0.376 AND
c12 < 0.4";
- let actual = execute(&mut ctx, sql);
- let expected =
"\"e\"\t0.39144436569161134\n\"d\"\t0.38870280983958583\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected =
"\"e\"\t0.39144436569161134\n\"d\"\t0.38870280983958583".to_string();
assert_eq!(expected, actual);
}
@@ -133,40 +135,39 @@ fn csv_query_with_predicate() {
fn csv_query_group_by_int_min_max() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
- //TODO add ORDER BY once supported, to make this test determistic
let sql = "SELECT c2, MIN(c12), MAX(c12) FROM aggregate_test_100 GROUP BY
c2";
- let actual = execute(&mut ctx, sql);
- let expected =
"4\t0.02182578039211991\t0.9237877978193884\n5\t0.01479305307777301\t0.9723580396501548\n2\t0.16301110515739792\t0.991517828651004\n3\t0.047343434291126085\t0.9293883502480845\n1\t0.05636955101974106\t0.9965400387585364\n".to_string();
- assert_eq!(expected, actual);
+ let mut actual = execute(&mut ctx, sql);
+ actual.sort();
+ let expected =
"1\t0.05636955101974106\t0.9965400387585364\n2\t0.16301110515739792\t0.991517828651004\n3\t0.047343434291126085\t0.9293883502480845\n4\t0.02182578039211991\t0.9237877978193884\n5\t0.01479305307777301\t0.9723580396501548".to_string();
+ assert_eq!(expected, actual.join("\n"));
}
#[test]
fn csv_query_avg() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
- //TODO add ORDER BY once supported, to make this test determistic
let sql = "SELECT avg(c12) FROM aggregate_test_100";
- let actual = execute(&mut ctx, sql);
- let expected = "0.5089725099127211\n".to_string();
- assert_eq!(expected, actual);
+ let mut actual = execute(&mut ctx, sql);
+ actual.sort();
+ let expected = "0.5089725099127211".to_string();
+ assert_eq!(expected, actual.join("\n"));
}
#[test]
fn csv_query_group_by_avg() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
- //TODO add ORDER BY once supported, to make this test determistic
let sql = "SELECT c1, avg(c12) FROM aggregate_test_100 GROUP BY c1";
- let actual = execute(&mut ctx, sql);
- let expected =
"\"a\"\t0.48754517466109415\n\"e\"\t0.48600669271341534\n\"d\"\t0.48855379387549824\n\"c\"\t0.6600456536439784\n\"b\"\t0.41040709263815384\n".to_string();
- assert_eq!(expected, actual);
+ let mut actual = execute(&mut ctx, sql);
+ actual.sort();
+ let expected =
"\"a\"\t0.48754517466109415\n\"b\"\t0.41040709263815384\n\"c\"\t0.6600456536439784\n\"d\"\t0.48855379387549824\n\"e\"\t0.48600669271341534".to_string();
+ assert_eq!(expected, actual.join("\n"));
}
#[test]
fn csv_query_avg_multi_batch() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
- //TODO add ORDER BY once supported, to make this test determistic
let sql = "SELECT avg(c12) FROM aggregate_test_100";
let plan = ctx.create_logical_plan(&sql).unwrap();
let plan = ctx.optimize(&plan).unwrap();
@@ -182,49 +183,14 @@ fn csv_query_avg_multi_batch() {
assert!((expected - actual).abs() < 0.01);
}
-//#[test]
-//fn csv_query_group_by_avg_multi_batch() {
-// let mut ctx = ExecutionContext::new();
-// register_aggregate_csv(&mut ctx);
-// //TODO add ORDER BY once supported, to make this test determistic
-// let sql = "SELECT c1, avg(c12) FROM aggregate_test_100 GROUP BY c1";
-// let plan = ctx.create_logical_plan(&sql).unwrap();
-// let results = ctx.execute(&plan, 4).unwrap();
-// let mut relation = results.borrow_mut();
-// let mut actual_vec = Vec::new();
-// while let Some(batch) = relation.next().unwrap() {
-// let column = batch.column(1);
-// let array = column.as_any().downcast_ref::<Float64Array>().unwrap();
-//
-// for row_index in 0..batch.num_rows() {
-// actual_vec.push(array.value(row_index));
-// }
-// }
-//
-// let expect_vec = vec![0.41040709, 0.48754517, 0.48855379, 0.66004565];
-//
-// actual_vec.sort_by(|a, b| a.partial_cmp(b).unwrap());
-//
-// println!("actual : {:?}", actual_vec);
-// println!("expected: {:?}", expect_vec);
-//
-// actual_vec
-// .iter()
-// .zip(expect_vec.iter())
-// .for_each(|(actual, expect)| {
-// // Due to float number's accuracy, different batch size will
lead to
-// different answers. assert!((expect - actual).abs() < 0.01);
-// });
-//}
-
#[test]
fn csv_query_count() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
//TODO add ORDER BY once supported, to make this test determistic
let sql = "SELECT count(c12) FROM aggregate_test_100";
- let actual = execute(&mut ctx, sql);
- let expected = "100\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected = "100".to_string();
assert_eq!(expected, actual);
}
@@ -234,8 +200,8 @@ fn csv_query_group_by_int_count() {
register_aggregate_csv(&mut ctx);
//TODO add ORDER BY once supported, to make this test determistic
let sql = "SELECT count(c12) FROM aggregate_test_100 GROUP BY c1";
- let actual = execute(&mut ctx, sql);
- let expected =
"\"a\"\t21\n\"e\"\t21\n\"d\"\t18\n\"c\"\t21\n\"b\"\t19\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected =
"\"a\"\t21\n\"e\"\t21\n\"d\"\t18\n\"c\"\t21\n\"b\"\t19".to_string();
assert_eq!(expected, actual);
}
@@ -245,9 +211,9 @@ fn csv_query_group_by_string_min_max() {
register_aggregate_csv(&mut ctx);
//TODO add ORDER BY once supported, to make this test determistic
let sql = "SELECT c2, MIN(c12), MAX(c12) FROM aggregate_test_100 GROUP BY
c1";
- let actual = execute(&mut ctx, sql);
+ let actual = execute(&mut ctx, sql).join("\n");
let expected =
-
"\"a\"\t0.02182578039211991\t0.9800193410444061\n\"e\"\t0.01479305307777301\t0.9965400387585364\n\"d\"\t0.061029375346466685\t0.9748360509016578\n\"c\"\t0.0494924465469434\t0.991517828651004\n\"b\"\t0.04893135681998029\t0.9185813970744787\n".to_string();
+
"\"a\"\t0.02182578039211991\t0.9800193410444061\n\"e\"\t0.01479305307777301\t0.9965400387585364\n\"d\"\t0.061029375346466685\t0.9748360509016578\n\"c\"\t0.0494924465469434\t0.991517828651004\n\"b\"\t0.04893135681998029\t0.9185813970744787".to_string();
assert_eq!(expected, actual);
}
@@ -256,8 +222,8 @@ fn csv_query_cast() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
let sql = "SELECT CAST(c12 AS float) FROM aggregate_test_100 WHERE c12 >
0.376 AND c12 < 0.4";
- let actual = execute(&mut ctx, sql);
- let expected = "0.39144436569161134\n0.38870280983958583\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected = "0.39144436569161134\n0.38870280983958583".to_string();
assert_eq!(expected, actual);
}
@@ -266,8 +232,8 @@ fn csv_query_cast_literal() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
let sql = "SELECT c12, CAST(1 AS float) FROM aggregate_test_100 WHERE c12
> CAST(0 AS float) LIMIT 2";
- let actual = execute(&mut ctx, sql);
- let expected =
"0.9294097332465232\t1.0\n0.3114712539863804\t1.0\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected =
"0.9294097332465232\t1.0\n0.3114712539863804\t1.0".to_string();
assert_eq!(expected, actual);
}
@@ -276,8 +242,8 @@ fn csv_query_limit() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
let sql = "SELECT 0 FROM aggregate_test_100 LIMIT 2";
- let actual = execute(&mut ctx, sql);
- let expected = "0\n0\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected = "0\n0".to_string();
assert_eq!(expected, actual);
}
@@ -286,8 +252,8 @@ fn csv_query_limit_bigger_than_nbr_of_rows() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
let sql = "SELECT c2 FROM aggregate_test_100 LIMIT 200";
- let actual = execute(&mut ctx, sql);
- let expected =
"2\n5\n1\n1\n5\n4\n3\n3\n1\n4\n1\n4\n3\n2\n1\n1\n2\n1\n3\n2\n4\n1\n5\n4\n2\n1\n4\n5\n2\n3\n4\n2\n1\n5\n3\n1\n2\n3\n3\n3\n2\n4\n1\n3\n2\n5\n2\n1\n4\n1\n4\n2\n5\n4\n2\n3\n4\n4\n4\n5\n4\n2\n1\n2\n4\n2\n3\n5\n1\n1\n4\n2\n1\n2\n1\n1\n5\n4\n5\n2\n3\n2\n4\n1\n3\n4\n3\n2\n5\n3\n3\n2\n5\n5\n4\n1\n3\n3\n4\n4\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected =
"2\n5\n1\n1\n5\n4\n3\n3\n1\n4\n1\n4\n3\n2\n1\n1\n2\n1\n3\n2\n4\n1\n5\n4\n2\n1\n4\n5\n2\n3\n4\n2\n1\n5\n3\n1\n2\n3\n3\n3\n2\n4\n1\n3\n2\n5\n2\n1\n4\n1\n4\n2\n5\n4\n2\n3\n4\n4\n4\n5\n4\n2\n1\n2\n4\n2\n3\n5\n1\n1\n4\n2\n1\n2\n1\n1\n5\n4\n5\n2\n3\n2\n4\n1\n3\n4\n3\n2\n5\n3\n3\n2\n5\n5\n4\n1\n3\n3\n4\n4".to_string();
assert_eq!(expected, actual);
}
@@ -296,8 +262,8 @@ fn csv_query_limit_with_same_nbr_of_rows() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
let sql = "SELECT c2 FROM aggregate_test_100 LIMIT 100";
- let actual = execute(&mut ctx, sql);
- let expected =
"2\n5\n1\n1\n5\n4\n3\n3\n1\n4\n1\n4\n3\n2\n1\n1\n2\n1\n3\n2\n4\n1\n5\n4\n2\n1\n4\n5\n2\n3\n4\n2\n1\n5\n3\n1\n2\n3\n3\n3\n2\n4\n1\n3\n2\n5\n2\n1\n4\n1\n4\n2\n5\n4\n2\n3\n4\n4\n4\n5\n4\n2\n1\n2\n4\n2\n3\n5\n1\n1\n4\n2\n1\n2\n1\n1\n5\n4\n5\n2\n3\n2\n4\n1\n3\n4\n3\n2\n5\n3\n3\n2\n5\n5\n4\n1\n3\n3\n4\n4\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected =
"2\n5\n1\n1\n5\n4\n3\n3\n1\n4\n1\n4\n3\n2\n1\n1\n2\n1\n3\n2\n4\n1\n5\n4\n2\n1\n4\n5\n2\n3\n4\n2\n1\n5\n3\n1\n2\n3\n3\n3\n2\n4\n1\n3\n2\n5\n2\n1\n4\n1\n4\n2\n5\n4\n2\n3\n4\n4\n4\n5\n4\n2\n1\n2\n4\n2\n3\n5\n1\n1\n4\n2\n1\n2\n1\n1\n5\n4\n5\n2\n3\n2\n4\n1\n3\n4\n3\n2\n5\n3\n3\n2\n5\n5\n4\n1\n3\n3\n4\n4".to_string();
assert_eq!(expected, actual);
}
@@ -306,7 +272,7 @@ fn csv_query_limit_zero() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv(&mut ctx);
let sql = "SELECT 0 FROM aggregate_test_100 LIMIT 0";
- let actual = execute(&mut ctx, sql);
+ let actual = execute(&mut ctx, sql).join("\n");
let expected = "".to_string();
assert_eq!(expected, actual);
}
@@ -316,8 +282,8 @@ fn csv_query_create_external_table() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv_by_sql(&mut ctx);
let sql = "SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, 10, c11, c12, c13
FROM aggregate_test_100 LIMIT 1";
- let actual = execute(&mut ctx, sql);
- let expected =
"\"c\"\t2\t1\t18109\t2033001162\t-6513304855495910254\t25\t43062\t1491205016\t10\t0.110830784\t0.9294097332465232\t\"6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW\"\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected =
"\"c\"\t2\t1\t18109\t2033001162\t-6513304855495910254\t25\t43062\t1491205016\t10\t0.110830784\t0.9294097332465232\t\"6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW\"".to_string();
assert_eq!(expected, actual);
}
@@ -326,24 +292,10 @@ fn csv_query_external_table_count() {
let mut ctx = ExecutionContext::new();
register_aggregate_csv_by_sql(&mut ctx);
let sql = "SELECT COUNT(c12) FROM aggregate_test_100";
- let actual = execute(&mut ctx, sql);
- let expected = "100\n".to_string();
- assert_eq!(expected, actual);
-}
-
-//TODO Uncomment the following test when ORDER BY is implemented to be able to
test ORDER
-// BY + LIMIT
-/*
-#[test]
-fn csv_query_limit_with_order_by() {
- let mut ctx = ExecutionContext::new();
- register_aggregate_csv(&mut ctx);
- let sql = "SELECT c7 FROM aggregate_test_100 ORDER BY c7 ASC LIMIT 2";
- let actual = execute(&mut ctx, sql);
- let expected = "0\n2\n".to_string();
+ let actual = execute(&mut ctx, sql).join("\n");
+ let expected = "100".to_string();
assert_eq!(expected, actual);
}
-*/
fn aggr_test_schema() -> Arc<Schema> {
Arc::new(Schema::new(vec![
@@ -427,7 +379,7 @@ fn register_alltypes_parquet(ctx: &mut ExecutionContext) {
}
/// Execute query and return result set as tab delimited string
-fn execute(ctx: &mut ExecutionContext, sql: &str) -> String {
+fn execute(ctx: &mut ExecutionContext, sql: &str) -> Vec<String> {
let plan = ctx.create_logical_plan(&sql).unwrap();
let plan = ctx.optimize(&plan).unwrap();
let plan = ctx.create_physical_plan(&plan, DEFAULT_BATCH_SIZE).unwrap();
@@ -435,10 +387,11 @@ fn execute(ctx: &mut ExecutionContext, sql: &str) ->
String {
result_str(&results)
}
-fn result_str(results: &Vec<RecordBatch>) -> String {
- let mut str = String::new();
+fn result_str(results: &Vec<RecordBatch>) -> Vec<String> {
+ let mut result = vec![];
for batch in results {
for row_index in 0..batch.num_rows() {
+ let mut str = String::new();
for column_index in 0..batch.num_columns() {
if column_index > 0 {
str.push_str("\t");
@@ -502,8 +455,8 @@ fn result_str(results: &Vec<RecordBatch>) -> String {
_ => str.push_str("???"),
}
}
- str.push_str("\n");
+ result.push(str);
}
}
- str
+ result
}
