kosiew commented on code in PR #74:
URL: https://github.com/apache/datafusion-spark/pull/74#discussion_r4583436051
##########
crates/spark/src/function/datetime/date_from_unix_date.rs:
##########
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, Date32Array};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::types::{NativeType, logical_int32, logical_string};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, internal_err};
+use datafusion_expr::{
+ Coercion, ColumnarValue, Documentation, ReturnFieldArgs,
ScalarFunctionArgs,
+ ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `date_from_unix_date` expression.
+///
+///
<https://spark.apache.org/docs/latest/api/sql/index.html#date_from_unix_date>
+#[user_doc(
+ doc_section(label = "Time and Date Functions"),
+ description = "Returns the date that is `days` days after 1970-01-01. This
is the inverse of `unix_date`. \
+ DataFusion has no built-in equivalent. Negative values give dates
before the epoch. \
+ Any INT is a valid date. Other numeric and string inputs are cast to
INT first, which \
+ truncates fractions and raises an error if the value is out of the INT
range or is \
+ not a valid number. Returns NULL if the input is NULL.",
+ syntax_example = "date_from_unix_date(days)",
+ sql_example = r#"```sql
+> SELECT date_from_unix_date(1000);
+1972-09-27
+> SELECT date_from_unix_date(-1);
+1969-12-31
+> SELECT date_from_unix_date(NULL);
+NULL
+```"#,
+ argument(
+ name = "days",
+ description = "Number of days since 1970-01-01, as an INT. Other
integer, floating-point, decimal and string inputs are cast to INT. BOOLEAN is
rejected. NULL returns NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkDateFromUnixDate {
+ signature: Signature,
+}
+
+impl Default for SparkDateFromUnixDate {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkDateFromUnixDate {
+ pub fn new() -> Self {
+ // Spark declares the argument as INT and implicitly casts other
numeric
+ // and string types to it. DataFusion types bare integer literals as
BIGINT.
+ let int32 = Coercion::new_implicit(
+ TypeSignatureClass::Native(logical_int32()),
+ vec![
+ TypeSignatureClass::Integer,
+ TypeSignatureClass::Float,
+ TypeSignatureClass::Decimal,
+ TypeSignatureClass::Native(logical_string()),
+ ],
+ NativeType::Int32,
+ );
+ Self {
+ signature: Signature::coercible(vec![int32],
Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkDateFromUnixDate {
+ fn name(&self) -> &str {
+ "date_from_unix_date"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, args: ReturnFieldArgs) ->
Result<FieldRef> {
+ let nullable = args.arg_fields[0].is_nullable();
Review Comment:
Optional: Could we add schema assertions for nullable and NOT NULL INT
columns, verifying that the Date32 output preserves input nullability? The
current tests check values and types but would miss a regression to
unconditional nullability. `DESCRIBE SELECT`, as used in the `concat_ws` tests,
should work.
##########
crates/spark/src/function/datetime/date_from_unix_date.rs:
##########
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, Date32Array};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::types::{NativeType, logical_int32, logical_string};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, internal_err};
+use datafusion_expr::{
+ Coercion, ColumnarValue, Documentation, ReturnFieldArgs,
ScalarFunctionArgs,
+ ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `date_from_unix_date` expression.
+///
+///
<https://spark.apache.org/docs/latest/api/sql/index.html#date_from_unix_date>
+#[user_doc(
+ doc_section(label = "Time and Date Functions"),
+ description = "Returns the date that is `days` days after 1970-01-01. This
is the inverse of `unix_date`. \
+ DataFusion has no built-in equivalent. Negative values give dates
before the epoch. \
+ Any INT is a valid date. Other numeric and string inputs are cast to
INT first, which \
Review Comment:
Optional: Could we clarify that these cast errors match Spark with ANSI mode
enabled? With ANSI mode disabled, Spark wraps overflowing BIGINTs and returns
NULL for malformed strings, while DataFusion still errors. A reference to
https://github.com/apache/datafusion-spark/issues/88 would document this known
divergence without implying that the UDF implements ANSI-off semantics.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]