uros-b commented on code in PR #58418:
URL: https://github.com/apache/spark/pull/58418#discussion_r3924722023
##########
python/pyspark/sql/types.py:
##########
@@ -484,6 +488,142 @@ def fromInternal(self, ts: int) -> datetime.datetime:
)
+class AnyTimestampNanoType(DatetimeType):
+ """
+ Super class of the nanosecond-capable timestamp data types
+ :class:`TimestampNTZNanosType` and :class:`TimestampLTZNanosType`.
+
+ .. versionadded:: 5.0.0
Review Comment:
```suggestion
.. versionadded:: 4.4.0
```
##########
python/pyspark/sql/types.py:
##########
@@ -484,6 +488,142 @@ def fromInternal(self, ts: int) -> datetime.datetime:
)
+class AnyTimestampNanoType(DatetimeType):
+ """
+ Super class of the nanosecond-capable timestamp data types
+ :class:`TimestampNTZNanosType` and :class:`TimestampLTZNanosType`.
+
+ .. versionadded:: 5.0.0
+ """
+
+ MIN_PRECISION: int = 7
+ MAX_PRECISION: int = 9
+ DEFAULT_PRECISION: int = 9
+ # Precision of the standard microsecond timestamp types, which the
parameterized DDL / JSON
+ # type names also accept (``timestamp_ntz(6)`` / ``timestamp_ltz(6)``).
+ MICROS_PRECISION: int = 6
+
+ # Set by each subclass to the SQL type name used in the DDL / JSON
representation, e.g.
+ # "timestamp_ntz". Also used, upper-cased, in the invalid-precision error
message.
+ _sqlTypeName: str = ""
+
+ def __init__(self, precision: int = DEFAULT_PRECISION):
+ # Reject non-integer precision (e.g. 7.5 or float("nan")), which would
otherwise slip
+ # through the range comparison below. operator.index accepts any
integer-like value
+ # (including a NumPy integer) and rejects the rest with a TypeError.
+ try:
+ precision = operator.index(precision)
+ except TypeError:
+ raise PySparkValueError(
+ errorClass="INVALID_TIMESTAMP_PRECISION",
+ messageParameters={
+ "precision": repr(precision),
+ "type": self._sqlTypeName.upper(),
+ },
+ )
+ if precision < self.MIN_PRECISION or precision > self.MAX_PRECISION:
+ raise PySparkValueError(
+ errorClass="INVALID_TIMESTAMP_PRECISION",
+ messageParameters={
+ "precision": str(precision),
+ "type": self._sqlTypeName.upper(),
+ },
+ )
+ self.precision = precision
+
+ def needConversion(self) -> bool:
+ return True
+
+ def simpleString(self) -> str:
+ return "%s(%d)" % (self._sqlTypeName, self.precision)
+
+ def jsonValue(self) -> str:
+ return "%s(%d)" % (self._sqlTypeName, self.precision)
+
+ def __repr__(self) -> str:
+ return "%s(%d)" % (type(self).__name__, self.precision)
+
+
+class TimestampNTZNanosType(AnyTimestampNanoType):
+ """Timestamp (datetime.datetime) data type without timezone information,
with
+ nanosecond-capable fractional-second precision (7 to 9 digits).
+
+ Parameters
+ ----------
+ precision : int, optional
+ Number of digits of fractional seconds, one of 7, 8 or 9 (default: 9).
+
+ Notes
+ -----
+ These types are behind the ``spark.sql.timestampNanosTypes.enabled``
preview flag (disabled
+ by default); using them while it is off raises an error.
+
+ ``datetime.datetime`` is microsecond-resolution, so values crossing the
Python boundary as
+ ``datetime.datetime`` -- :meth:`DataFrame.collect`,
:meth:`DataFrame.toLocalIterator`, and
+ Python UDF arguments -- are truncated to microseconds, as are
``datetime.datetime`` values
+ supplied to :meth:`SparkSession.createDataFrame` from Python lists/rows.
The value stored by
+ Spark keeps full precision; only this Python boundary is
microsecond-resolution. A ``map``
+ with keys of this type that differ only below a microsecond would collapse
to one entry, so
+ that conversion raises rather than silently dropping an entry.
+
+ Arrow- and pandas-based conversion for these types --
:meth:`DataFrame.toPandas`,
+ :meth:`SparkSession.createDataFrame` from a pandas ``DataFrame``,
Arrow-based UDFs, and the
+ Spark Connect data path -- is not yet supported and raises
+ ``UNSUPPORTED_DATA_TYPE_FOR_ARROW_CONVERSION``; it is planned as a
follow-up.
+
+ .. versionadded:: 5.0.0
Review Comment:
```suggestion
.. versionadded:: 4.4.0
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
