andygrove commented on code in PR #74:
URL: https://github.com/apache/datafusion-spark/pull/74#discussion_r4585461657


##########
crates/spark/src/function/datetime/date_from_unix_date.rs:
##########
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, Date32Array};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::types::{NativeType, logical_int32, logical_string};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, internal_err};
+use datafusion_expr::{
+    Coercion, ColumnarValue, Documentation, ReturnFieldArgs, 
ScalarFunctionArgs,
+    ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `date_from_unix_date` expression.
+///
+/// 
<https://spark.apache.org/docs/latest/api/sql/index.html#date_from_unix_date>
+#[user_doc(
+    doc_section(label = "Time and Date Functions"),
+    description = "Returns the date that is `days` days after 1970-01-01. This 
is the inverse of `unix_date`. \
+        DataFusion has no built-in equivalent. Negative values give dates 
before the epoch. \
+        Any INT is a valid date. Other numeric and string inputs are cast to 
INT first, which \
+        truncates fractions and raises an error if the value is out of the INT 
range or is \
+        not a valid number. Returns NULL if the input is NULL.",
+    syntax_example = "date_from_unix_date(days)",
+    sql_example = r#"```sql
+> SELECT date_from_unix_date(1000);
+1972-09-27
+> SELECT date_from_unix_date(-1);
+1969-12-31
+> SELECT date_from_unix_date(NULL);
+NULL
+```"#,
+    argument(
+        name = "days",
+        description = "Number of days since 1970-01-01, as an INT. Other 
integer, floating-point, decimal and string inputs are cast to INT. BOOLEAN is 
rejected. NULL returns NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkDateFromUnixDate {
+    signature: Signature,
+}
+
+impl Default for SparkDateFromUnixDate {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkDateFromUnixDate {
+    pub fn new() -> Self {
+        // Spark declares the argument as INT and implicitly casts other 
numeric
+        // and string types to it. DataFusion types bare integer literals as 
BIGINT.
+        let int32 = Coercion::new_implicit(
+            TypeSignatureClass::Native(logical_int32()),
+            vec![
+                TypeSignatureClass::Integer,
+                TypeSignatureClass::Float,
+                TypeSignatureClass::Decimal,
+                TypeSignatureClass::Native(logical_string()),
+            ],
+            NativeType::Int32,
+        );
+        Self {
+            signature: Signature::coercible(vec![int32], 
Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkDateFromUnixDate {
+    fn name(&self) -> &str {
+        "date_from_unix_date"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+        internal_err!("return_field_from_args should be used instead")
+    }
+
+    fn return_field_from_args(&self, args: ReturnFieldArgs) -> 
Result<FieldRef> {
+        let nullable = args.arg_fields[0].is_nullable();

Review Comment:
   Added in 6a55064. A `DESCRIBE SELECT` over a NOT NULL INT column and a 
nullable one checks that the result is `Date32 NO` and `Date32 YES`, matching 
the Spark 4.2.0 schema.



##########
crates/spark/src/function/datetime/date_from_unix_date.rs:
##########
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, Date32Array};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::types::{NativeType, logical_int32, logical_string};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, internal_err};
+use datafusion_expr::{
+    Coercion, ColumnarValue, Documentation, ReturnFieldArgs, 
ScalarFunctionArgs,
+    ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `date_from_unix_date` expression.
+///
+/// 
<https://spark.apache.org/docs/latest/api/sql/index.html#date_from_unix_date>
+#[user_doc(
+    doc_section(label = "Time and Date Functions"),
+    description = "Returns the date that is `days` days after 1970-01-01. This 
is the inverse of `unix_date`. \
+        DataFusion has no built-in equivalent. Negative values give dates 
before the epoch. \
+        Any INT is a valid date. Other numeric and string inputs are cast to 
INT first, which \

Review Comment:
   Done in 6a55064. The description now says the errors match Spark with ANSI 
mode enabled, and that with ANSI mode disabled Spark wraps an out-of-range 
BIGINT and returns NULL for a malformed string while we still error, with a 
link to #88. I checked both ANSI-off results in PySpark 4.2.0.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to