This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 667ec30  [SPARK-61069][CONNECT][PYTHON] Collect TIMESTAMP_NTZ values 
as wall-clock times and timestamps exactly, and raise NOT_IMPLEMENTED for 
year-month intervals, as PySpark does
667ec30 is described below

commit 667ec30ba20485e3af1789e8e5f31eeb44c6ac0a
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 12:28:37 2026 +0900

    [SPARK-61069][CONNECT][PYTHON] Collect TIMESTAMP_NTZ values as wall-clock 
times and timestamps exactly, and raise NOT_IMPLEMENTED for year-month 
intervals, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    - The Arrow decoder keeps TIMESTAMP_NTZ values apart, as 
`Value::TimestampNtz`, from an Arrow timestamp without a time zone, as Spark's 
`ArrowUtils` declares NTZ (`crates/spark-connect/src/dataframe.rs`, `row.rs`; 
the encoder, literals and inference take the new variant too). The Python 
conversion makes an NTZ value the wall clock, `datetime(1970, 1, 1) + 
timedelta(microseconds=...)`, in any client time zone, nested in arrays, maps 
and structs too (`crates/pyspark-rs/src/row.rs`).
    - TIMESTAMP values convert with exact integer arithmetic to the client's 
local time, as PySpark's conversion does, instead of `datetime.fromtimestamp` 
of a float, which lost a microsecond for some values.
    - `collect`, `first`, `head`, `take` and `toLocalIterator` raise 
`PySparkNotImplementedError` `NOT_IMPLEMENTED` ("Collecting a year-month 
interval value in Spark Connect") for a schema with a year-month interval, 
nested or not, empty results included, as PySpark does 
(`crates/pyspark-rs/src/dataframe.rs`). The Rust core still renders such values 
as strings.
    
    ### Why are the changes needed?
    
    [SPARK-61069](https://issues.apache.org/jira/browse/SPARK-61069): against a 
local Spark 4.3.0 server (session time zone UTC):
    
    | Code (client `TZ=America/Los_Angeles` for the first two) | Current master 
| This change and PySpark 4.3.0 |
    | --- | --- | --- |
    | `spark.sql("select timestamp_ntz'2020-07-01 12:00:00' t").collect()` | 
`datetime(2020, 7, 1, 5, 0)` | `datetime(2020, 7, 1, 12, 0)` |
    | an NTZ `0001-01-01` | `ValueError: year 0 is out of range` | `datetime(1, 
1, 1, 0, 0)` |
    | `spark.sql("select timestamp_micros(9223372036854775) t").collect()` 
(UTC) | `...16.854776` | `...16.854775` |
    | `spark.sql("SELECT INTERVAL '1-2' YEAR TO MONTH ym").collect()` | 
`[Row(ym='1-2')]` | `PySparkNotImplementedError [NOT_IMPLEMENTED]` |
    
    1,812 random and extreme microsecond values, in three time zones, as 
TIMESTAMP and as TIMESTAMP_NTZ, now collect identically to PySpark 4.3.0.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, these values collect as in PySpark.
    
    ### How was this patch tested?
    
    - New offline tests in `python/tests/test_dropin_offline.py` for NTZ values 
in several time zones and years, nested; exact TIMESTAMP values; and year-month 
intervals through every collecting API; live tests in 
`python/tests/test_dropin_e2e.py` with `TZ=America/Los_Angeles` and for 
year-month intervals; and decoder unit tests for NTZ.
    - The lib tests of `apache-spark-connect` (458 passed), the offline suite 
(949 passed, 8 skipped), and the live suite.
    
    Unchanged: `toArrow()` of a year-month interval returns a table where 
PySpark raises `UNSUPPORTED_DATA_TYPE_FOR_ARROW_CONVERSION`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #259 from HyukjinKwon/SPARK-61069-collect-ntz-intervals.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs    |  62 ++++++++++++--
 crates/pyspark-rs/src/row.rs          |  48 +++++++++--
 crates/pyspark-rs/src/session.rs      |   1 +
 crates/spark-connect/src/dataframe.rs |  87 ++++++++++++++++----
 crates/spark-connect/src/plan.rs      |   1 +
 crates/spark-connect/src/row.rs       |   7 +-
 crates/spark-connect/src/session.rs   |  12 ++-
 python/tests/test_dropin_e2e.py       |  72 +++++++++++++++++
 python/tests/test_dropin_offline.py   | 148 ++++++++++++++++++++++++++++++++++
 9 files changed, 405 insertions(+), 33 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 23e3ca9..235bd46 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -3516,6 +3516,50 @@ fn udt_field_converters(
     Ok(Some(std::sync::Arc::new(converters.extract()?)))
 }
 
+/// Whether `data_type` holds a year-month interval, also nested in an array, 
a map, a struct or
+/// a user-defined type (`pyspark.sql.types._has_type`).
+fn has_year_month_interval(data_type: &spark_connect::types::DataType) -> bool 
{
+    use spark_connect::types::DataType;
+    match data_type {
+        DataType::YearMonthInterval { .. } => true,
+        DataType::Struct { fields } => fields
+            .iter()
+            .any(|field| has_year_month_interval(&field.data_type)),
+        DataType::Array { element_type, .. } => 
has_year_month_interval(element_type),
+        DataType::Map {
+            key_type,
+            value_type,
+            ..
+        } => has_year_month_interval(key_type) || 
has_year_month_interval(value_type),
+        DataType::Udt {
+            sql_type: Some(sql_type),
+            ..
+        } => has_year_month_interval(sql_type),
+        _ => false,
+    }
+}
+
+/// Raises as PySpark's Arrow conversion of the rows does for a result of 
`schema` with a
+/// year-month interval, which Spark Connect cannot collect: the error is 
raised even when the
+/// result has no rows.
+fn check_collectable(
+    py: Python<'_>,
+    schema: Option<&spark_connect::types::DataType>,
+) -> PyResult<()> {
+    if schema.is_some_and(has_year_month_interval) {
+        return Err(crate::column::raise_pyspark(
+            py,
+            "PySparkNotImplementedError",
+            "NOT_IMPLEMENTED",
+            &[(
+                "feature",
+                "Collecting a year-month interval value in Spark Connect",
+            )],
+        ));
+    }
+    Ok(())
+}
+
 /// The collected `rows` of a result of `schema`, whose values of user-defined 
types are
 /// deserialized when first read.
 fn rows_to_py(
@@ -3523,6 +3567,7 @@ fn rows_to_py(
     rows: Vec<spark_connect::row::Row>,
     schema: Option<&spark_connect::types::DataType>,
 ) -> PyResult<Vec<PyRow>> {
+    check_collectable(py, schema)?;
     let converters = udt_field_converters(py, schema)?;
     Ok(rows
         .into_iter()
@@ -3584,12 +3629,19 @@ impl PyLocalRowIterator {
             return Ok(None);
         }
         // Release the GIL while fetching from the Rust iterator (which may 
block on network I/O).
-        match py.detach(|| self.next_row()) {
-            Some(Ok(row)) => {
-                if self.converters.is_none() {
-                    let schema = 
self.iterator.as_ref().and_then(LocalRowIterator::schema);
-                    self.converters = Some(udt_field_converters(py, 
schema.as_ref())?);
+        let next = py.detach(|| self.next_row());
+        if self.converters.is_none() {
+            if let Some(schema) = 
self.iterator.as_ref().and_then(LocalRowIterator::schema) {
+                // The result is known once its first message arrives, 
possibly with no rows.
+                if let Err(error) = check_collectable(py, Some(&schema)) {
+                    self.finished = true;
+                    return Err(error);
                 }
+                self.converters = Some(udt_field_converters(py, 
Some(&schema))?);
+            }
+        }
+        match next {
+            Some(Ok(row)) => {
                 let converters = self.converters.clone().flatten();
                 Ok(Some(PyRow::with_converters(row, converters)))
             }
diff --git a/crates/pyspark-rs/src/row.rs b/crates/pyspark-rs/src/row.rs
index 669a073..c39773f 100644
--- a/crates/pyspark-rs/src/row.rs
+++ b/crates/pyspark-rs/src/row.rs
@@ -178,6 +178,35 @@ impl PyRow {
     }
 }
 
+/// `datetime.datetime(1970, 1, 1)`, the epoch of Spark's timestamps, without 
a time zone.
+fn naive_epoch(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
+    py.import("datetime")?
+        .getattr("datetime")?
+        .call1((1970, 1, 1))
+}
+
+/// `datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)`, the epoch 
of the instants.
+fn utc_epoch(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
+    let datetime = py.import("datetime")?;
+    let kwargs = PyDict::new(py);
+    kwargs.set_item("tzinfo", datetime.getattr("timezone")?.getattr("utc")?)?;
+    datetime
+        .getattr("datetime")?
+        .call((1970, 1, 1), Some(&kwargs))
+}
+
+/// The `datetime.timedelta` of `micros` microseconds, exact over the whole 
i64 range: a float
+/// of seconds would lose the microseconds of the values far from the epoch.
+fn micros_timedelta(py: Python<'_>, micros: i64) -> PyResult<Bound<'_, PyAny>> 
{
+    let days = micros.div_euclid(86_400_000_000);
+    let remainder = micros.rem_euclid(86_400_000_000);
+    py.import("datetime")?.getattr("timedelta")?.call1((
+        days,
+        remainder / 1_000_000,
+        remainder % 1_000_000,
+    ))
+}
+
 /// Convert a Spark `Value` into a Python object (PyO3 0.28 `IntoPyObject`).
 pub fn value_to_py<'py>(py: Python<'py>, v: &Value) -> PyResult<Bound<'py, 
PyAny>> {
     // Propagate conversion failures as Python exceptions with `?` rather than
@@ -202,15 +231,18 @@ pub fn value_to_py<'py>(py: Python<'py>, v: &Value) -> 
PyResult<Bound<'py, PyAny
             let ordinal = (*d as i64) + 719163i64;
             date_cls.call_method1("fromordinal", (ordinal,))?.into_any()
         }
+        // TIMESTAMP (with a time zone): the instant, shown in the local time 
zone of the client,
+        // as PySpark's Arrow conversion does (`astimezone()` of the UTC 
datetime, naive).
         Value::Timestamp(t) => {
-            // Convert microseconds-since-epoch (i64) to datetime.datetime
-            let datetime_mod = py.import("datetime")?;
-            let datetime_cls = datetime_mod.getattr("datetime")?;
-            let seconds = *t as f64 / 1_000_000.0;
-            datetime_cls
-                .call_method1("fromtimestamp", (seconds,))?
-                .into_any()
-        }
+            let utc = utc_epoch(py)?.add(micros_timedelta(py, *t)?)?;
+            let local = utc.call_method0("astimezone")?;
+            let kwargs = PyDict::new(py);
+            kwargs.set_item("tzinfo", py.None())?;
+            local.call_method("replace", (), Some(&kwargs))?.into_any()
+        }
+        // TIMESTAMP_NTZ (without a time zone): the wall clock, which no time 
zone shifts, as
+        // PySpark's `TimestampNTZType.fromInternal` gives it.
+        Value::TimestampNtz(t) => naive_epoch(py)?.add(micros_timedelta(py, 
*t)?)?,
         Value::Time(nanos) => {
             // As PySpark's `TimeType.fromInternal`: a datetime.time of the 
microseconds.
             let (hours, remainder) = (
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index a6694fd..06ea7cd 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1858,6 +1858,7 @@ fn value_to_datatype(v: &Value) -> DataType {
         Value::Binary(_) => DataType::Binary,
         Value::Date(_) => DataType::Date,
         Value::Timestamp(_) => DataType::Timestamp,
+        Value::TimestampNtz(_) => DataType::TimestampNtz,
         // As PySpark infers TimeType() of a datetime.time.
         Value::Time(_) => DataType::Time { precision: 6 },
         Value::DayTimeInterval(_) => DataType::DayTimeInterval {
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index 55a3762..5666c67 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -2469,6 +2469,17 @@ struct ArrowValueLayout<'a> {
     decoder: ArrowValueDecoder<'a>,
 }
 
+/// The Value of a timestamp of `micros` since the epoch. Spark's Arrow types 
declare
+/// TIMESTAMP_NTZ without a time zone and TIMESTAMP with one 
(`ArrowUtils.toArrowType`), so the
+/// time zone of the Arrow type tells the two apart.
+fn timestamp_value(micros: i64, has_time_zone: bool) -> Value {
+    if has_time_zone {
+        Value::Timestamp(micros)
+    } else {
+        Value::TimestampNtz(micros)
+    }
+}
+
 enum ArrowValueDecoder<'a> {
     Null,
     Boolean(&'a arrow::array::BooleanArray),
@@ -2656,7 +2667,9 @@ impl<'a> ArrowValueLayout<'a> {
             ArrowValueDecoder::String(arr) => 
Ok(Value::String(arr.value(index).to_string())),
             ArrowValueDecoder::Binary(arr) => 
Ok(Value::Binary(arr.value(index).to_vec())),
             ArrowValueDecoder::Date32(arr) => 
Ok(Value::Date(arr.value(index))),
-            ArrowValueDecoder::TimestampMicrosecond(arr) => 
Ok(Value::Timestamp(arr.value(index))),
+            ArrowValueDecoder::TimestampMicrosecond(arr) => {
+                Ok(timestamp_value(arr.value(index), arr.timezone().is_some()))
+            }
             ArrowValueDecoder::UInt8(arr) => Ok(Value::Short(arr.value(index) 
as i16)),
             ArrowValueDecoder::UInt16(arr) => 
Ok(Value::Integer(arr.value(index) as i32)),
             ArrowValueDecoder::UInt32(arr) => Ok(Value::Long(arr.value(index) 
as i64)),
@@ -2680,15 +2693,18 @@ impl<'a> ArrowValueLayout<'a> {
             ArrowValueDecoder::LargeBinary(arr) => 
Ok(Value::Binary(arr.value(index).to_vec())),
             ArrowValueDecoder::StringView(arr) => 
Ok(Value::String(arr.value(index).to_string())),
             ArrowValueDecoder::BinaryView(arr) => 
Ok(Value::Binary(arr.value(index).to_vec())),
-            ArrowValueDecoder::TimestampSecond(arr) => {
-                Ok(Value::Timestamp(arr.value(index) * 1_000_000))
-            }
-            ArrowValueDecoder::TimestampMillisecond(arr) => {
-                Ok(Value::Timestamp(arr.value(index) * 1_000))
-            }
-            ArrowValueDecoder::TimestampNanosecond(arr) => {
-                Ok(Value::Timestamp(arr.value(index) / 1_000))
-            }
+            ArrowValueDecoder::TimestampSecond(arr) => Ok(timestamp_value(
+                arr.value(index) * 1_000_000,
+                arr.timezone().is_some(),
+            )),
+            ArrowValueDecoder::TimestampMillisecond(arr) => Ok(timestamp_value(
+                arr.value(index) * 1_000,
+                arr.timezone().is_some(),
+            )),
+            ArrowValueDecoder::TimestampNanosecond(arr) => Ok(timestamp_value(
+                arr.value(index) / 1_000,
+                arr.timezone().is_some(),
+            )),
             ArrowValueDecoder::Date64(arr) => {
                 Ok(Value::Date((arr.value(index) / 86_400_000) as i32))
             }
@@ -4539,17 +4555,26 @@ mod arrow_value_tests {
         };
         let cases: Vec<(ArrayRef, Value)> = vec![
             (
-                Arc::new(TimestampSecondArray::from(vec![-1])),
+                
Arc::new(TimestampSecondArray::from(vec![-1]).with_timezone_utc()),
                 Value::Timestamp(-1_000_000),
             ),
             (
-                Arc::new(TimestampMillisecondArray::from(vec![-1])),
+                
Arc::new(TimestampMillisecondArray::from(vec![-1]).with_timezone_utc()),
                 Value::Timestamp(-1_000),
             ),
             (
-                Arc::new(TimestampNanosecondArray::from(vec![-1999])),
+                
Arc::new(TimestampNanosecondArray::from(vec![-1999]).with_timezone_utc()),
                 Value::Timestamp(-1),
             ),
+            // Without a time zone, as TIMESTAMP_NTZ is declared.
+            (
+                Arc::new(TimestampSecondArray::from(vec![-1])),
+                Value::TimestampNtz(-1_000_000),
+            ),
+            (
+                Arc::new(TimestampMicrosecondArray::from(vec![-1])),
+                Value::TimestampNtz(-1),
+            ),
             (
                 Arc::new(Date64Array::from(vec![-86_400_001])),
                 Value::Date(-1),
@@ -4649,6 +4674,14 @@ mod arrow_value_tests {
         ));
         assert!(matches!(
             arrow_value_at(&TimestampMicrosecondArray::from(vec![1i64]), 
0).unwrap(),
+            Value::TimestampNtz(1)
+        ));
+        assert!(matches!(
+            arrow_value_at(
+                
&TimestampMicrosecondArray::from(vec![1i64]).with_timezone_utc(),
+                0
+            )
+            .unwrap(),
             Value::Timestamp(1)
         ));
     }
@@ -4715,14 +4748,38 @@ mod arrow_value_tests {
     fn timestamps_and_date64() {
         assert!(matches!(
             arrow_value_at(&TimestampSecondArray::from(vec![1i64]), 
0).unwrap(),
-            Value::Timestamp(_)
+            Value::TimestampNtz(_)
         ));
         assert!(matches!(
             arrow_value_at(&TimestampMillisecondArray::from(vec![1i64]), 
0).unwrap(),
-            Value::Timestamp(_)
+            Value::TimestampNtz(_)
         ));
         assert!(matches!(
             arrow_value_at(&TimestampNanosecondArray::from(vec![1000i64]), 
0).unwrap(),
+            Value::TimestampNtz(_)
+        ));
+        assert!(matches!(
+            arrow_value_at(
+                &TimestampSecondArray::from(vec![1i64]).with_timezone_utc(),
+                0
+            )
+            .unwrap(),
+            Value::Timestamp(_)
+        ));
+        assert!(matches!(
+            arrow_value_at(
+                
&TimestampMillisecondArray::from(vec![1i64]).with_timezone_utc(),
+                0
+            )
+            .unwrap(),
+            Value::Timestamp(_)
+        ));
+        assert!(matches!(
+            arrow_value_at(
+                
&TimestampNanosecondArray::from(vec![1000i64]).with_timezone_utc(),
+                0
+            )
+            .unwrap(),
             Value::Timestamp(_)
         ));
         assert!(matches!(
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index a8341cf..bef5092 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -2008,6 +2008,7 @@ fn value_to_proto_literal(v: &crate::row::Value) -> 
proto::expression::Literal {
         Value::String(s) => LiteralType::String(s.clone()),
         Value::Date(d) => LiteralType::Date(*d),
         Value::Timestamp(t) => LiteralType::Timestamp(*t),
+        Value::TimestampNtz(t) => LiteralType::TimestampNtz(*t),
         // Spark rejects interval NA fills. Preserve the type so it errors 
instead
         // of silently filling string columns with the Value's Debug text.
         Value::DayTimeInterval(t) => LiteralType::DayTimeInterval(*t),
diff --git a/crates/spark-connect/src/row.rs b/crates/spark-connect/src/row.rs
index 219f916..b7e9559 100644
--- a/crates/spark-connect/src/row.rs
+++ b/crates/spark-connect/src/row.rs
@@ -31,8 +31,11 @@ pub enum Value {
     Binary(Vec<u8>),
     /// Date (days since epoch)
     Date(i32),
-    /// Timestamp (microseconds since epoch)
+    /// Timestamp with local time zone (`TimestampType`), in microseconds 
since the epoch.
     Timestamp(i64),
+    /// Timestamp without time zone (`TimestampNTZType`), in microseconds of 
its wall clock
+    /// since 1970-01-01 00:00:00, which the time zone of the client does not 
shift.
+    TimestampNtz(i64),
     /// TIME, in nanoseconds since midnight, as PySpark's internal value of 
`TimeType`.
     Time(i64),
     /// Spark day-time interval, in signed microseconds. The full i64 range is
@@ -74,6 +77,7 @@ impl fmt::Debug for Value {
             Self::Binary(v) => f.debug_tuple("Binary").field(v).finish(),
             Self::Date(v) => f.debug_tuple("Date").field(v).finish(),
             Self::Timestamp(v) => f.debug_tuple("Timestamp").field(v).finish(),
+            Self::TimestampNtz(v) => 
f.debug_tuple("TimestampNtz").field(v).finish(),
             Self::Time(v) => f.debug_tuple("Time").field(v).finish(),
             Self::DayTimeInterval(v) => 
f.debug_tuple("DayTimeInterval").field(v).finish(),
             Self::Decimal {
@@ -177,6 +181,7 @@ impl fmt::Display for Value {
             Value::Binary(b) => write!(f, "{:?}", b),
             Value::Date(d) => write!(f, "{}", d),
             Value::Timestamp(t) => write!(f, "{}", t),
+            Value::TimestampNtz(t) => write!(f, "{}", t),
             Value::Time(t) => write!(f, "{}", t),
             Value::DayTimeInterval(t) => write!(f, "{}", t),
             Value::Decimal {
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index e75c0c6..d5adfc8 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -1288,7 +1288,7 @@ fn values_to_arrow(values: &[&Value], dt: &DataType) -> 
Result<Arc<dyn arrow::ar
                 .iter()
                 .map(|v| match *v {
                     Value::Null => Ok(None),
-                    Value::Timestamp(t) => Ok(Some(*t)),
+                    Value::Timestamp(t) | Value::TimestampNtz(t) => 
Ok(Some(*t)),
                     _ => Err(SparkError::connect_msg("Type mismatch in row 
data")),
                 })
                 .collect();
@@ -1307,9 +1307,13 @@ fn values_to_arrow(values: &[&Value], dt: &DataType) -> 
Result<Arc<dyn arrow::ar
                 .iter()
                 .map(|v| match *v {
                     Value::Null => Ok(None),
-                    Value::Timestamp(t) => 
t.checked_mul(1_000).map(Some).ok_or_else(|| {
-                        SparkError::connect_msg("Timestamp out of range for 
nanosecond precision")
-                    }),
+                    Value::Timestamp(t) | Value::TimestampNtz(t) => {
+                        t.checked_mul(1_000).map(Some).ok_or_else(|| {
+                            SparkError::connect_msg(
+                                "Timestamp out of range for nanosecond 
precision",
+                            )
+                        })
+                    }
                     _ => Err(SparkError::connect_msg("Type mismatch in row 
data")),
                 })
                 .collect();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 89bc02c..66125a4 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -235,6 +235,78 @@ def 
test_replace_and_unpivot_take_their_arguments_as_pyspark(session):
     assert wide.melt("id", "x", "var", "val").collect() == expected[:1]
 
 
+def _run_in_time_zone(code, zone):
+    # Runs `code` in a new interpreter whose time zone is `zone`: the local 
time zone of the
+    # client, which the test process (UTC) does not have.
+    import subprocess
+    import sys
+
+    env = dict(os.environ, TZ=zone)
+    result = subprocess.run(
+        [sys.executable, "-c", code], env=env, capture_output=True, text=True, 
timeout=300)
+    assert result.returncode == 0, result.stderr
+    return result.stdout
+
+
+def test_ntz_collect_is_the_wall_clock_in_the_client_time_zone_as_pyspark():
+    # As PySpark's Arrow conversion: a TIMESTAMP_NTZ is the wall clock, 
whatever the time zone of
+    # the client, from year 1 to year 9999 and inside arrays, maps and 
structs, while a TIMESTAMP
+    # is the instant, shown in the local time zone (here seven hours behind 
UTC, in July).
+    remote = os.environ.get("SPARK_REMOTE")
+    if not remote:
+        pytest.skip("SPARK_REMOTE is required for live-server tests")
+    out = _run_in_time_zone("""
+import datetime
+from pyspark.sql import SparkSession
+spark = SparkSession.builder.remote(%r).getOrCreate()
+print(spark.createDataFrame([(datetime.datetime(2020, 7, 1, 12, 0),)], "t 
timestamp_ntz").collect())
+print(spark.sql("select timestamp_ntz'2020-07-01 12:00:00' t").collect())
+print(spark.sql("select timestamp_ntz'0001-01-01 00:00:00' t, 
timestamp_ntz'9999-12-31 23:59:59.999999' u").collect())
+print(spark.sql(
+    "select array(timestamp_ntz'2020-07-01 12:00:00') a, map('k', 
timestamp_ntz'2020-07-01 12:00:00') m, "
+    "named_struct('x', timestamp_ntz'2020-07-01 12:00:00') s").collect())
+print(spark.sql("select timestamp'2020-07-01 12:00:00' t, 
timestamp_micros(9223372036854775) u").collect())
+""" % remote, "America/Los_Angeles")
+    lines = out.strip().splitlines()
+    assert lines[0] == "[Row(t=datetime.datetime(2020, 7, 1, 12, 0))]"
+    assert lines[1] == "[Row(t=datetime.datetime(2020, 7, 1, 12, 0))]"
+    assert lines[2] == ("[Row(t=datetime.datetime(1, 1, 1, 0, 0), "
+                        "u=datetime.datetime(9999, 12, 31, 23, 59, 59, 
999999))]")
+    assert lines[3] == ("[Row(a=[datetime.datetime(2020, 7, 1, 12, 0)], "
+                        "m={'k': datetime.datetime(2020, 7, 1, 12, 0)}, "
+                        "s=Row(x=datetime.datetime(2020, 7, 1, 12, 0)))]")
+    assert lines[4] == ("[Row(t=datetime.datetime(2020, 7, 1, 5, 0), "
+                        "u=datetime.datetime(2262, 4, 11, 16, 47, 16, 
854775))]")
+
+
+def test_year_month_intervals_are_not_collected_as_pyspark(session):
+    # As PySpark's Arrow conversion of the rows: a year-month interval, also 
nested, is not
+    # collected in Spark Connect, and raises NOT_IMPLEMENTED, even for an 
empty result.
+    import datetime
+
+    from pyspark.errors import PySparkNotImplementedError
+
+    message = (r"\[NOT_IMPLEMENTED\] Collecting a year-month interval value in 
Spark Connect "
+               r"is not implemented\.")
+    df = session.sql("SELECT INTERVAL '1-2' YEAR TO MONTH ym")
+    for collect in (
+        lambda d: d.collect(),
+        lambda d: d.first(),
+        lambda d: d.head(1),
+        lambda d: d.take(1),
+        lambda d: list(d.toLocalIterator()),
+        lambda d: d.where("1 = 0").collect(),
+        lambda d: session.sql("SELECT array(INTERVAL '1-2' YEAR TO MONTH) 
a").collect(),
+        lambda d: session.sql("SELECT map('a', INTERVAL '1-2' YEAR TO MONTH) 
m").collect(),
+        lambda d: session.sql("SELECT named_struct('a', INTERVAL '1-2' YEAR TO 
MONTH) s").collect(),
+    ):
+        with pytest.raises(PySparkNotImplementedError, match=message):
+            collect(df)
+    # A day-time interval is collected as a timedelta.
+    assert session.sql("SELECT INTERVAL '1 02:03:04' DAY TO SECOND 
dt").collect()[0][0] == (
+        datetime.timedelta(days=1, seconds=7384))
+
+
 def test_instr_and_regexp_replace_apply_their_optional_arguments(session):
     # Column b holds "c", so the literal "b" and the column b find different 
positions.
     df = session.createDataFrame([("abcabc", "c", "a", "z")], ["s", "b", "p", 
"r"])
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index f35818c..723c3ce 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7,7 +7,9 @@ that require a live server live in test_dropin_e2e.py and are 
gated on SPARK_REM
 """
 
 import array
+import contextlib
 import datetime
+import os
 import re
 import decimal
 import json
@@ -20,6 +22,7 @@ from threading import Barrier
 import pytest
 
 from _mock_connect_server import (
+    ddl_data_type,
     ddl_json,
     ddl_parse_response,
     message_field,
@@ -8276,6 +8279,151 @@ def test_create_auto_cdc_flow_scd2_track_history():
     assert flow.spark_conf == {"k": "v"}
 
 
+# --------------------------------------------------------------------------- 
collect of NTZ timestamps, distant timestamps and year-month intervals
+
+
[email protected]
+def _client_time_zone(name):
+    # The process time zone, which the local conversions of datetime read, for 
a block.
+    previous = os.environ.get("TZ")
+    os.environ["TZ"] = name
+    time.tzset()
+    try:
+        yield
+    finally:
+        if previous is None:
+            os.environ.pop("TZ", None)
+        else:
+            os.environ["TZ"] = previous
+        time.tzset()
+
+
+def _collect_mock_result(mock_connect, table, schema_ddl):
+    # Makes the mock server answer the next execution with `table` (Arrow IPC) 
of the type
+    # `schema_ddl`: ExecutePlanResponse.schema=7, arrow_batch=2 
(ArrowBatch.row_count=1, data=2).
+    import pyarrow as pa
+
+    spark, service = mock_connect
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, table.schema) as writer:
+        writer.write_table(table)
+    batch = varint_field(1, table.num_rows) + message_field(2, 
sink.getvalue().to_pybytes())
+    service.execute_payload = message_field(7, ddl_data_type(schema_ddl)) + 
message_field(2, batch)
+    return spark.range(1)
+
+
+def _micros_since_epoch(value):
+    # The microseconds since 1970-01-01 00:00:00 of a naive datetime, exactly.
+    return (value - datetime.datetime(1970, 1, 1)) // 
datetime.timedelta(microseconds=1)
+
+
+def _naive_from_micros(micros):
+    # The naive datetime of `micros` microseconds since the epoch, as integer 
arithmetic.
+    return datetime.datetime(1970, 1, 1) + 
datetime.timedelta(microseconds=micros)
+
+
+_NTZ_VALUES = [
+    datetime.datetime(1, 1, 1),
+    datetime.datetime(1500, 3, 1, 1, 2, 3, 4),
+    datetime.datetime(2020, 7, 1, 12, 0),
+    datetime.datetime(9999, 12, 31, 23, 59, 59, 999999),
+]
+
+
+def 
test_ntz_collect_is_the_wall_clock_in_any_time_zone_as_pyspark(mock_connect):
+    # TIMESTAMP_NTZ (no time zone in Arrow) is the wall clock, which the time 
zone of the client
+    # does not shift, from year 1 to year 9999, in arrays, maps and structs 
too.
+    import pyarrow as pa
+
+    ntz = pa.timestamp("us")
+    values = [_micros_since_epoch(v) for v in _NTZ_VALUES]
+    table = pa.table({
+        "t": pa.array(values, ntz),
+        "a": pa.array([[v] for v in values], pa.list_(ntz)),
+        "m": pa.array([[("k", v)] for v in values], pa.map_(pa.string(), ntz)),
+        "s": pa.array([{"x": v} for v in values], pa.struct([("x", ntz)])),
+    })
+    df = _collect_mock_result(
+        mock_connect, table,
+        "t timestamp_ntz, a array<timestamp_ntz>, m map<string,timestamp_ntz>, 
"
+        "s struct<x:timestamp_ntz>")
+    for zone in ("UTC", "America/Los_Angeles", "Asia/Kolkata"):
+        with _client_time_zone(zone):
+            rows = df.collect()
+            assert [r.t for r in rows] == _NTZ_VALUES
+            assert [r.a for r in rows] == [[v] for v in _NTZ_VALUES]
+            assert [r.m for r in rows] == [{"k": v} for v in _NTZ_VALUES]
+            assert [r.s for r in rows] == [Row(x=v) for v in _NTZ_VALUES]
+
+
+def 
test_ltz_collect_is_the_instant_in_the_client_time_zone_as_pyspark(mock_connect):
+    # TIMESTAMP (Arrow with a time zone) is the instant, shown in the local 
time zone of the
+    # client, with its microseconds exact however far it is from the epoch.
+    import pyarrow as pa
+
+    micros = [
+        _micros_since_epoch(datetime.datetime(2020, 7, 1, 12, 0)),
+        _micros_since_epoch(datetime.datetime(2020, 7, 1, 12, 0, 0, 1)),
+        9223372036854775,
+        -9223372036854775,
+        253402300799999999,
+        -62135596800000000,
+        1,
+        -1,
+    ]
+    table = pa.table({"t": pa.array(micros, pa.timestamp("us", tz="UTC"))})
+    df = _collect_mock_result(mock_connect, table, "t timestamp")
+    with _client_time_zone("UTC"):
+        assert [r.t for r in df.collect()] == [_naive_from_micros(m) for m in 
micros]
+    # Year 1 is out of the range of a local time in Los Angeles (as in 
PySpark), so it is left out.
+    in_range = [m for m in micros if m != -62135596800000000]
+    df = _collect_mock_result(
+        mock_connect, pa.table({"t": pa.array(in_range, pa.timestamp("us", 
tz="UTC"))}), "t timestamp")
+    with _client_time_zone("America/Los_Angeles"):
+        rows = [r.t for r in df.collect()]
+        assert rows[0] == datetime.datetime(2020, 7, 1, 5, 0)
+        assert rows[1] == datetime.datetime(2020, 7, 1, 5, 0, 0, 1)
+        # 2262-04-11 23:47:16.854775 UTC is PDT, seven hours earlier.
+        assert rows[2] == datetime.datetime(2262, 4, 11, 16, 47, 16, 854775)
+
+
[email protected]("collect", [
+    lambda df: df.collect(),
+    lambda df: df.first(),
+    lambda df: df.head(1),
+    lambda df: df.take(1),
+    lambda df: list(df.toLocalIterator()),
+])
[email protected]("schema_ddl, table_type", [
+    ("ym interval year to month", "int32"),
+    ("a array<interval year to month>", "list"),
+    ("m map<string,interval year to month>", "map"),
+    ("s struct<ym:interval year to month>", "struct"),
+])
+def test_year_month_intervals_are_not_collected_as_pyspark(
+        mock_connect, collect, schema_ddl, table_type):
+    # PySpark's Arrow conversion of the rows raises NOT_IMPLEMENTED for a 
year-month interval
+    # anywhere in the schema, before any row is converted, so an empty result 
raises too.
+    # The Arrow data is empty: Spark Connect has no pyarrow type to build a 
year-month one.
+    import pyarrow as pa
+    from pyspark.errors import PySparkNotImplementedError
+
+    arrow_type = {
+        "int32": pa.int32(),
+        "list": pa.list_(pa.int32()),
+        "map": pa.map_(pa.string(), pa.int32()),
+        "struct": pa.struct([("ym", pa.int32())]),
+    }[table_type]
+    column = "ym" if table_type == "int32" else ("s" if table_type == "struct" 
else
+                                                  ("m" if table_type == "map" 
else "a"))
+    table = pa.table({column: pa.array([], arrow_type)})
+    df = _collect_mock_result(mock_connect, table, schema_ddl)
+    with pytest.raises(PySparkNotImplementedError, match=(
+            r"\[NOT_IMPLEMENTED\] Collecting a year-month interval value in 
Spark Connect "
+            r"is not implemented\.")):
+        collect(df)
+
+
 # ---------------------------------------------------------------- DataFrame 
repr
 
 def _repr_varint(value):


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to