This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 11a20bf ARROW-8854: [Rust] [Integration Testing] Standardize error
handling
11a20bf is described below
commit 11a20bf7f2e796277f11d7e995ee80613f646466
Author: Andy Grove <[email protected]>
AuthorDate: Tue May 19 18:49:02 2020 -0600
ARROW-8854: [Rust] [Integration Testing] Standardize error handling
With these changes, I can now see why tests are failing, and also all test
failures are now being detected.
This also adds partial support for `DataType::Null`.
Current status:
```
################# FAILURES #################
FAILED TEST: primitive Java producing, Rust consuming
FAILED TEST: primitive_zerolength Java producing, Rust consuming
FAILED TEST: null Java producing, Rust consuming
FAILED TEST: null_trivial Java producing, Rust consuming
FAILED TEST: datetime Java producing, Rust consuming
FAILED TEST: null Rust producing, Java consuming
FAILED TEST: null_trivial Rust producing, Java consuming
FAILED TEST: datetime Rust producing, Java consuming
FAILED TEST: primitive_large_offsets Rust producing, Rust consuming
FAILED TEST: null Rust producing, Rust consuming
FAILED TEST: null_trivial Rust producing, Rust consuming
FAILED TEST: datetime Rust producing, Rust consuming
12 failures
```
Closes #7219 from andygrove/ARROW-8854
Authored-by: Andy Grove <[email protected]>
Signed-off-by: Andy Grove <[email protected]>
---
rust/arrow/src/datatypes.rs | 4 ++++
rust/arrow/src/ipc/convert.rs | 7 ++++++
.../src/bin/arrow-file-to-stream.rs | 9 +++----
.../src/bin/arrow-json-integration-test.rs | 28 +++++++++++-----------
.../src/bin/arrow-stream-to-file.rs | 15 ++++++------
rust/parquet/src/arrow/schema.rs | 1 +
6 files changed, 39 insertions(+), 25 deletions(-)
diff --git a/rust/arrow/src/datatypes.rs b/rust/arrow/src/datatypes.rs
index 138d551..d3e129c 100644
--- a/rust/arrow/src/datatypes.rs
+++ b/rust/arrow/src/datatypes.rs
@@ -57,6 +57,8 @@ use crate::error::{ArrowError, Result};
/// [the physical memory layout of Apache
Arrow](https://arrow.apache.org/docs/format/Columnar.html#physical-memory-layout).
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash,
PartialOrd, Ord)]
pub enum DataType {
+ /// Null type
+ Null,
/// A boolean datatype representing the values `true` and `false`.
Boolean,
/// A signed 8-bit integer.
@@ -763,6 +765,7 @@ impl DataType {
fn from(json: &Value) -> Result<DataType> {
match *json {
Value::Object(ref map) => match map.get("name") {
+ Some(s) if s == "null" => Ok(DataType::Null),
Some(s) if s == "bool" => Ok(DataType::Boolean),
Some(s) if s == "binary" => Ok(DataType::Binary),
Some(s) if s == "utf8" => Ok(DataType::Utf8),
@@ -925,6 +928,7 @@ impl DataType {
/// Generate a JSON representation of the data type
pub fn to_json(&self) -> Value {
match self {
+ DataType::Null => json!({"name": "null"}),
DataType::Boolean => json!({"name": "bool"}),
DataType::Int8 => json!({"name": "int", "bitWidth": 8, "isSigned":
true}),
DataType::Int16 => json!({"name": "int", "bitWidth": 16,
"isSigned": true}),
diff --git a/rust/arrow/src/ipc/convert.rs b/rust/arrow/src/ipc/convert.rs
index 953885f..84d6b39 100644
--- a/rust/arrow/src/ipc/convert.rs
+++ b/rust/arrow/src/ipc/convert.rs
@@ -194,6 +194,7 @@ pub(crate) fn get_data_type(field: ipc::Field,
may_be_dictionary: bool) -> DataT
}
match field.type_type() {
+ ipc::Type::Null => DataType::Null,
ipc::Type::Bool => DataType::Boolean,
ipc::Type::Int => {
let int = field.type_as_int().unwrap();
@@ -330,6 +331,11 @@ pub(crate) fn get_fb_field_type<'a: 'b, 'b>(
// An empty field list is thus returned for primitive types
let empty_fields: Vec<WIPOffset<ipc::Field>> = vec![];
match data_type {
+ Null => (
+ ipc::Type::Null,
+ ipc::NullBuilder::new(fbb).finish().as_union_value(),
+ None,
+ ),
Boolean => {
let children = fbb.create_vector(&empty_fields[..]);
(
@@ -609,6 +615,7 @@ mod tests {
Field::new("float16", DataType::Float16, true),
Field::new("float32", DataType::Float32, false),
Field::new("float64", DataType::Float64, true),
+ Field::new("null", DataType::Null, false),
Field::new("bool", DataType::Boolean, false),
Field::new("date32", DataType::Date32(DateUnit::Day), false),
Field::new("date64", DataType::Date64(DateUnit::Millisecond),
true),
diff --git a/rust/integration-testing/src/bin/arrow-file-to-stream.rs
b/rust/integration-testing/src/bin/arrow-file-to-stream.rs
index 980a617..1b6e360 100644
--- a/rust/integration-testing/src/bin/arrow-file-to-stream.rs
+++ b/rust/integration-testing/src/bin/arrow-file-to-stream.rs
@@ -17,7 +17,7 @@
use std::env;
use std::fs::File;
-use std::io::{self, BufReader, BufWriter};
+use std::io::{self, BufReader};
use arrow::error::Result;
use arrow::ipc::reader::FileReader;
@@ -25,7 +25,7 @@ use arrow::ipc::writer::StreamWriter;
fn main() -> Result<()> {
let args: Vec<String> = env::args().collect();
- eprintln!("Rust: arrow-file-to-stream; args={:?}", args);
+ eprintln!("{:?}", args);
let filename = &args[1];
eprintln!("Reading from Arrow file {}", filename);
@@ -35,13 +35,14 @@ fn main() -> Result<()> {
let mut reader = FileReader::try_new(reader)?;
let schema = reader.schema();
- let writer = BufWriter::new(io::stdout());
- let mut writer = StreamWriter::try_new(writer, &schema)?;
+ let mut writer = StreamWriter::try_new(io::stdout(), &schema)?;
while let Some(batch) = reader.next()? {
writer.write(&batch)?;
}
writer.finish()?;
+ eprintln!("Completed without error");
+
Ok(())
}
diff --git a/rust/integration-testing/src/bin/arrow-json-integration-test.rs
b/rust/integration-testing/src/bin/arrow-json-integration-test.rs
index 580d493..ab916ae 100644
--- a/rust/integration-testing/src/bin/arrow-json-integration-test.rs
+++ b/rust/integration-testing/src/bin/arrow-json-integration-test.rs
@@ -37,10 +37,11 @@ use std::fs::File;
use std::io::BufReader;
use std::sync::Arc;
-fn main() {
+fn main() -> Result<()> {
let args: Vec<String> = env::args().collect();
- println!("{:?}", args);
- let matches = App::new("rust arrow-json-integrationtest")
+ eprintln!("{:?}", args);
+
+ let matches = App::new("rust arrow-json-integration-test")
.arg(Arg::with_name("integration")
.long("integration"))
.arg(Arg::with_name("arrow")
@@ -68,24 +69,19 @@ fn main() {
.value_of("json")
.expect("must provide path to json file");
let mode = matches.value_of("mode").unwrap();
- let verbose = matches.value_of("verbose").is_some();
+ let verbose = true; //matches.value_of("verbose").is_some();
- let res = match mode {
+ match mode {
"JSON_TO_ARROW" => json_to_arrow(json_file, arrow_file, verbose),
"ARROW_TO_JSON" => arrow_to_json(arrow_file, json_file, verbose),
"VALIDATE" => validate(arrow_file, json_file, verbose),
_ => panic!(format!("mode {} not supported", mode)),
- };
-
- match res {
- Ok(_) => (),
- Err(e) => eprint!("error: {}", e),
}
}
fn json_to_arrow(json_name: &str, arrow_name: &str, verbose: bool) ->
Result<()> {
if verbose {
- println!("Converting {} to {}", json_name, arrow_name);
+ eprintln!("Converting {} to {}", json_name, arrow_name);
}
let (schema, batches) = read_json_file(json_name)?;
@@ -316,7 +312,7 @@ fn record_batch_from_json(
fn arrow_to_json(arrow_name: &str, json_name: &str, verbose: bool) ->
Result<()> {
if verbose {
- println!("Converting {} to {}", arrow_name, json_name);
+ eprintln!("Converting {} to {}", arrow_name, json_name);
}
let arrow_file = File::open(arrow_name)?;
@@ -347,7 +343,7 @@ fn arrow_to_json(arrow_name: &str, json_name: &str,
verbose: bool) -> Result<()>
fn validate(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()> {
if verbose {
- println!("Validating {} and {}", arrow_name, json_name);
+ eprintln!("Validating {} and {}", arrow_name, json_name);
}
// open JSON file
@@ -367,7 +363,11 @@ fn validate(arrow_name: &str, json_name: &str, verbose:
bool) -> Result<()> {
assert!(arrow_batch.num_columns() == json_batch.num_columns());
assert!(arrow_batch.num_rows() == json_batch.num_rows());
- // TODO compare in more detail
+ // TODO compare in more detail
+ eprintln!(
+ "Basic validation of {} and {} PASSES",
+ arrow_name, json_name
+ );
} else {
return Err(ArrowError::ComputeError(
"no more arrow batches left".to_owned(),
diff --git a/rust/integration-testing/src/bin/arrow-stream-to-file.rs
b/rust/integration-testing/src/bin/arrow-stream-to-file.rs
index 62d9d09..d61004b 100644
--- a/rust/integration-testing/src/bin/arrow-stream-to-file.rs
+++ b/rust/integration-testing/src/bin/arrow-stream-to-file.rs
@@ -15,27 +15,28 @@
// specific language governing permissions and limitations
// under the License.
-use std::io::BufWriter;
-use std::io::{self, BufReader};
+use std::env;
+use std::io;
use arrow::error::Result;
use arrow::ipc::reader::StreamReader;
use arrow::ipc::writer::FileWriter;
fn main() -> Result<()> {
- eprintln!("Rust: arrow-stream-to-file");
+ let args: Vec<String> = env::args().collect();
+ eprintln!("{:?}", args);
- let reader = BufReader::new(io::stdin());
- let mut arrow_stream_reader = StreamReader::try_new(reader)?;
+ let mut arrow_stream_reader = StreamReader::try_new(io::stdin())?;
let schema = arrow_stream_reader.schema();
- let writer = BufWriter::new(io::stdout());
- let mut writer = FileWriter::try_new(writer, &schema)?;
+ let mut writer = FileWriter::try_new(io::stdout(), &schema)?;
while let Some(batch) = arrow_stream_reader.next()? {
writer.write(&batch)?;
}
writer.finish()?;
+ eprintln!("Completed without error");
+
Ok(())
}
diff --git a/rust/parquet/src/arrow/schema.rs b/rust/parquet/src/arrow/schema.rs
index 8b9a90e..4f56bbb 100644
--- a/rust/parquet/src/arrow/schema.rs
+++ b/rust/parquet/src/arrow/schema.rs
@@ -142,6 +142,7 @@ fn arrow_to_parquet_type(field: &Field) -> Result<Type> {
};
// create type from field
match field.data_type() {
+ DataType::Null => Err(ArrowError("Null arrays not
supported".to_string())),
DataType::Boolean => Type::primitive_type_builder(name,
PhysicalType::BOOLEAN)
.with_repetition(repetition)
.build(),
