This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 4055c003c8 Minor: Fix `clippy` by switching to `timestamp_nanos_opt`
instead of (deprecated) `timestamp_nanos` (#7572)
4055c003c8 is described below
commit 4055c003c8a5a3ee672f48f09fe57b43432c3fdb
Author: Andrew Lamb <[email protected]>
AuthorDate: Fri Sep 15 12:36:15 2023 -0400
Minor: Fix `clippy` by switching to `timestamp_nanos_opt` instead of
(deprecated) `timestamp_nanos` (#7572)
* Use timestamp_nanos_opt instead of deprecated timestamp_nanos
* Update dependencies
---
datafusion-cli/Cargo.lock | 92 +++++++++++-----------
datafusion/common/src/scalar.rs | 3 +-
datafusion/core/src/physical_plan/metrics/value.rs | 6 +-
.../physical_plan/sorts/sort_preserving_merge.rs | 10 ++-
datafusion/core/tests/parquet/mod.rs | 3 +-
datafusion/core/tests/sql/explain_analyze.rs | 9 ++-
.../src/simplify_expressions/simplify_exprs.rs | 4 +-
.../physical-expr/src/datetime_expressions.rs | 9 ++-
datafusion/proto/src/physical_plan/to_proto.rs | 8 +-
9 files changed, 81 insertions(+), 63 deletions(-)
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index ac09bc7f83..51d36c5a90 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -77,9 +77,9 @@ dependencies = [
[[package]]
name = "anstyle"
-version = "1.0.2"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea"
+checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46"
[[package]]
name = "arrayref"
@@ -323,9 +323,9 @@ dependencies = [
[[package]]
name = "async-compression"
-version = "0.4.2"
+version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d495b6dc0184693324491a5ac05f559acc97bf937ab31d7a1c33dd0016be6d2b"
+checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c"
dependencies = [
"bzip2",
"flate2",
@@ -347,7 +347,7 @@ checksum =
"bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.31",
+ "syn 2.0.33",
]
[[package]]
@@ -674,9 +674,9 @@ dependencies = [
[[package]]
name = "base64"
-version = "0.21.3"
+version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53"
+checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
[[package]]
name = "base64-simd"
@@ -766,9 +766,9 @@ dependencies = [
[[package]]
name = "bumpalo"
-version = "3.13.0"
+version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
+checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
[[package]]
name = "byteorder"
@@ -831,9 +831,9 @@ checksum =
"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
-version = "0.4.30"
+version = "0.4.31"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877"
+checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
dependencies = [
"android-tzdata",
"iana-time-zone",
@@ -1031,7 +1031,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "1f34ba9a9bcb8645379e9de8cb3ecfcf4d1c85ba66d90deb3259206fa5aa193b"
dependencies = [
"quote",
- "syn 2.0.31",
+ "syn 2.0.33",
]
[[package]]
@@ -1503,7 +1503,7 @@ checksum =
"89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.31",
+ "syn 2.0.33",
]
[[package]]
@@ -1938,9 +1938,9 @@ dependencies = [
[[package]]
name = "libc"
-version = "0.2.147"
+version = "0.2.148"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
+checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
[[package]]
name = "libm"
@@ -1950,9 +1950,9 @@ checksum =
"f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"
[[package]]
name = "libmimalloc-sys"
-version = "0.1.34"
+version = "0.1.35"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "25d058a81af0d1c22d7a1c948576bee6d673f7af3c0f35564abd6c81122f513d"
+checksum = "3979b5c37ece694f1f5e51e7ecc871fdb0f517ed04ee45f88d15d6d553cb9664"
dependencies = [
"cc",
"libc",
@@ -1960,9 +1960,9 @@ dependencies = [
[[package]]
name = "linux-raw-sys"
-version = "0.4.5"
+version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
+checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128"
[[package]]
name = "lock_api"
@@ -2028,9 +2028,9 @@ checksum =
"8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
[[package]]
name = "mimalloc"
-version = "0.1.38"
+version = "0.1.39"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "972e5f23f6716f62665760b0f4cbf592576a80c7b879ba9beaafc0e558894127"
+checksum = "fa01922b5ea280a911e323e4d2fd24b7fe5cc4042e0d2cda3c40775cdc4bdc9c"
dependencies = [
"libmimalloc-sys",
]
@@ -2389,7 +2389,7 @@ checksum =
"4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.31",
+ "syn 2.0.33",
]
[[package]]
@@ -2479,9 +2479,9 @@ checksum =
"dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
[[package]]
name = "proc-macro2"
-version = "1.0.66"
+version = "1.0.67"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
+checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328"
dependencies = [
"unicode-ident",
]
@@ -2702,9 +2702,9 @@ dependencies = [
[[package]]
name = "rustix"
-version = "0.38.11"
+version = "0.38.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453"
+checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662"
dependencies = [
"bitflags 2.4.0",
"errno",
@@ -2760,9 +2760,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.101.4"
+version = "0.101.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d"
+checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed"
dependencies = [
"ring",
"untrusted",
@@ -2889,14 +2889,14 @@ checksum =
"4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.31",
+ "syn 2.0.33",
]
[[package]]
name = "serde_json"
-version = "1.0.105"
+version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360"
+checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65"
dependencies = [
"itoa",
"ryu",
@@ -2987,9 +2987,9 @@ dependencies = [
[[package]]
name = "socket2"
-version = "0.5.3"
+version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877"
+checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e"
dependencies = [
"libc",
"windows-sys",
@@ -3078,7 +3078,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.31",
+ "syn 2.0.33",
]
[[package]]
@@ -3100,9 +3100,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.31"
+version = "2.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398"
+checksum = "9caece70c63bfba29ec2fed841a09851b14a235c60010fa4de58089b6c025668"
dependencies = [
"proc-macro2",
"quote",
@@ -3160,7 +3160,7 @@ checksum =
"49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.31",
+ "syn 2.0.33",
]
[[package]]
@@ -3238,7 +3238,7 @@ dependencies = [
"num_cpus",
"parking_lot",
"pin-project-lite",
- "socket2 0.5.3",
+ "socket2 0.5.4",
"tokio-macros",
"windows-sys",
]
@@ -3251,7 +3251,7 @@ checksum =
"630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.31",
+ "syn 2.0.33",
]
[[package]]
@@ -3349,7 +3349,7 @@ checksum =
"5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.31",
+ "syn 2.0.33",
]
[[package]]
@@ -3379,9 +3379,9 @@ dependencies = [
[[package]]
name = "typenum"
-version = "1.16.0"
+version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
+checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]]
name = "unicode-bidi"
@@ -3391,9 +3391,9 @@ checksum =
"92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
[[package]]
name = "unicode-ident"
-version = "1.0.11"
+version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "unicode-normalization"
@@ -3521,7 +3521,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.31",
+ "syn 2.0.33",
"wasm-bindgen-shared",
]
@@ -3555,7 +3555,7 @@ checksum =
"54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.31",
+ "syn 2.0.33",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs
index fa2175c223..32343b98fa 100644
--- a/datafusion/common/src/scalar.rs
+++ b/datafusion/common/src/scalar.rs
@@ -4965,7 +4965,8 @@ mod tests {
.unwrap()
.and_hms_nano_opt(hour, minute, second, nanosec)
.unwrap()
- .timestamp_nanos(),
+ .timestamp_nanos_opt()
+ .unwrap(),
),
None,
))
diff --git a/datafusion/core/src/physical_plan/metrics/value.rs
b/datafusion/core/src/physical_plan/metrics/value.rs
index 59b012f25a..899ceb60b4 100644
--- a/datafusion/core/src/physical_plan/metrics/value.rs
+++ b/datafusion/core/src/physical_plan/metrics/value.rs
@@ -430,11 +430,13 @@ impl MetricValue {
Self::Time { time, .. } => time.value(),
Self::StartTimestamp(timestamp) => timestamp
.value()
- .map(|ts| ts.timestamp_nanos() as usize)
+ .and_then(|ts| ts.timestamp_nanos_opt())
+ .map(|nanos| nanos as usize)
.unwrap_or(0),
Self::EndTimestamp(timestamp) => timestamp
.value()
- .map(|ts| ts.timestamp_nanos() as usize)
+ .and_then(|ts| ts.timestamp_nanos_opt())
+ .map(|nanos| nanos as usize)
.unwrap_or(0),
}
}
diff --git a/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs
b/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs
index c9f693a8a2..fbad5acb01 100644
--- a/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs
+++ b/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs
@@ -283,7 +283,7 @@ mod tests {
use crate::physical_plan::coalesce_partitions::CoalescePartitionsExec;
use crate::physical_plan::expressions::col;
use crate::physical_plan::memory::MemoryExec;
- use crate::physical_plan::metrics::MetricValue;
+ use crate::physical_plan::metrics::{MetricValue, Timestamp};
use crate::physical_plan::sorts::sort::SortExec;
use crate::physical_plan::stream::RecordBatchReceiverStream;
use crate::physical_plan::{collect, common};
@@ -939,11 +939,11 @@ mod tests {
metrics.iter().for_each(|m| match m.value() {
MetricValue::StartTimestamp(ts) => {
saw_start = true;
- assert!(ts.value().unwrap().timestamp_nanos() > 0);
+ assert!(nanos_from_timestamp(ts) > 0);
}
MetricValue::EndTimestamp(ts) => {
saw_end = true;
- assert!(ts.value().unwrap().timestamp_nanos() > 0);
+ assert!(nanos_from_timestamp(ts) > 0);
}
_ => {}
});
@@ -952,6 +952,10 @@ mod tests {
assert!(saw_end);
}
+ fn nanos_from_timestamp(ts: &Timestamp) -> i64 {
+ ts.value().unwrap().timestamp_nanos_opt().unwrap()
+ }
+
#[tokio::test]
async fn test_drop_cancel() -> Result<()> {
let task_ctx = Arc::new(TaskContext::default());
diff --git a/datafusion/core/tests/parquet/mod.rs
b/datafusion/core/tests/parquet/mod.rs
index 6f289e0c06..db7349851b 100644
--- a/datafusion/core/tests/parquet/mod.rs
+++ b/datafusion/core/tests/parquet/mod.rs
@@ -290,7 +290,8 @@ fn make_timestamp_batch(offset: Duration) -> RecordBatch {
offset_nanos
+ t.parse::<chrono::NaiveDateTime>()
.unwrap()
- .timestamp_nanos()
+ .timestamp_nanos_opt()
+ .unwrap()
})
})
.collect::<Vec<_>>();
diff --git a/datafusion/core/tests/sql/explain_analyze.rs
b/datafusion/core/tests/sql/explain_analyze.rs
index f32ffc1642..06120c01ce 100644
--- a/datafusion/core/tests/sql/explain_analyze.rs
+++ b/datafusion/core/tests/sql/explain_analyze.rs
@@ -18,6 +18,7 @@
use super::*;
use datafusion::config::ConfigOptions;
use datafusion::physical_plan::display::DisplayableExecutionPlan;
+use datafusion::physical_plan::metrics::Timestamp;
#[tokio::test]
async fn explain_analyze_baseline_metrics() {
@@ -142,11 +143,11 @@ async fn explain_analyze_baseline_metrics() {
metrics.iter().for_each(|m| match m.value() {
MetricValue::StartTimestamp(ts) => {
saw_start = true;
- assert!(ts.value().unwrap().timestamp_nanos() > 0);
+ assert!(nanos_from_timestamp(ts) > 0);
}
MetricValue::EndTimestamp(ts) => {
saw_end = true;
- assert!(ts.value().unwrap().timestamp_nanos() > 0);
+ assert!(nanos_from_timestamp(ts) > 0);
}
_ => {}
});
@@ -161,7 +162,9 @@ async fn explain_analyze_baseline_metrics() {
datafusion::physical_plan::accept(physical_plan.as_ref(), &mut
TimeValidator {})
.unwrap();
}
-
+fn nanos_from_timestamp(ts: &Timestamp) -> i64 {
+ ts.value().unwrap().timestamp_nanos_opt().unwrap()
+}
#[tokio::test]
async fn csv_explain_plans() {
// This test verify the look of each plan in its full cycle plan creation
diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs
b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs
index e6d66720ee..59c1c5a126 100644
--- a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs
+++ b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs
@@ -473,8 +473,8 @@ mod tests {
let expected = format!(
"Projection: TimestampNanosecond({}, Some(\"+00:00\")) AS now(),
TimestampNanosecond({}, Some(\"+00:00\")) AS t2\
\n TableScan: test",
- time.timestamp_nanos(),
- time.timestamp_nanos()
+ time.timestamp_nanos_opt().unwrap(),
+ time.timestamp_nanos_opt().unwrap()
);
assert_eq!(expected, actual);
diff --git a/datafusion/physical-expr/src/datetime_expressions.rs
b/datafusion/physical-expr/src/datetime_expressions.rs
index bf90a7da9c..63f7645fea 100644
--- a/datafusion/physical-expr/src/datetime_expressions.rs
+++ b/datafusion/physical-expr/src/datetime_expressions.rs
@@ -168,7 +168,7 @@ pub fn to_timestamp_seconds(args: &[ColumnarValue]) ->
Result<ColumnarValue> {
pub fn make_now(
now_ts: DateTime<Utc>,
) -> impl Fn(&[ColumnarValue]) -> Result<ColumnarValue> {
- let now_ts = Some(now_ts.timestamp_nanos());
+ let now_ts = now_ts.timestamp_nanos_opt();
move |_arg| {
Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
now_ts,
@@ -204,7 +204,7 @@ pub fn make_current_date(
pub fn make_current_time(
now_ts: DateTime<Utc>,
) -> impl Fn(&[ColumnarValue]) -> Result<ColumnarValue> {
- let nano = Some(now_ts.timestamp_nanos() % 86400000000000);
+ let nano = now_ts.timestamp_nanos_opt().map(|ts| ts % 86400000000000);
move |_arg| Ok(ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(nano)))
}
@@ -271,8 +271,9 @@ fn date_trunc_coarse(granularity: &str, value: i64) ->
Result<i64> {
return exec_err!("Unsupported date_trunc granularity:
{unsupported}");
}
};
+ let value = value.and_then(|value| value.timestamp_nanos_opt());
// `with_x(0)` are infallible because `0` are always a valid
- Ok(value.unwrap().timestamp_nanos())
+ Ok(value.unwrap())
}
// truncates a single value with the given timeunit to the specified
granularity
@@ -459,7 +460,7 @@ fn date_bin_months_interval(stride_months: i64, source:
i64, origin: i64) -> i64
};
}
- bin_time.timestamp_nanos()
+ bin_time.timestamp_nanos_opt().unwrap()
}
fn to_utc_date_time(nanos: i64) -> DateTime<Utc> {
diff --git a/datafusion/proto/src/physical_plan/to_proto.rs
b/datafusion/proto/src/physical_plan/to_proto.rs
index 62221f9d75..a5b1300360 100644
--- a/datafusion/proto/src/physical_plan/to_proto.rs
+++ b/datafusion/proto/src/physical_plan/to_proto.rs
@@ -422,10 +422,16 @@ impl TryFrom<&PartitionedFile> for
protobuf::PartitionedFile {
type Error = DataFusionError;
fn try_from(pf: &PartitionedFile) -> Result<Self, Self::Error> {
+ let last_modified = pf.object_meta.last_modified;
+ let last_modified_ns =
last_modified.timestamp_nanos_opt().ok_or_else(|| {
+ DataFusionError::Plan(format!(
+ "Invalid timestamp on PartitionedFile::ObjectMeta:
{last_modified}"
+ ))
+ })? as u64;
Ok(protobuf::PartitionedFile {
path: pf.object_meta.location.as_ref().to_owned(),
size: pf.object_meta.size as u64,
- last_modified_ns: pf.object_meta.last_modified.timestamp_nanos()
as u64,
+ last_modified_ns,
partition_values: pf
.partition_values
.iter()
