zhengruifeng commented on code in PR #46733:
URL: https://github.com/apache/spark/pull/46733#discussion_r1613083917
##########
python/pyspark/sql/types.py:
##########
@@ -1756,13 +1756,45 @@ def toJson(self, zone_id: str = "UTC") -> str:
TimestampNTZType,
NullType,
VariantType,
+ YearMonthIntervalType,
+ DayTimeIntervalType,
]
-_all_atomic_types: Dict[str, Type[DataType]] = dict((t.typeName(), t) for t in
_atomic_types)
-_complex_types: List[Type[Union[ArrayType, MapType, StructType]]] =
[ArrayType, MapType, StructType]
-_all_complex_types: Dict[str, Type[Union[ArrayType, MapType, StructType]]] =
dict(
- (v.typeName(), v) for v in _complex_types
-)
+_complex_types: List[Type[Union[ArrayType, MapType, StructType]]] = [
+ ArrayType,
+ MapType,
+ StructType,
+]
+_all_complex_types: Dict[str, Type[Union[ArrayType, MapType, StructType]]] = {
+ "array": ArrayType,
+ "map": MapType,
+ "struct": StructType,
+}
+
+# Datatypes that can be directly parsed by mapping a json string without regex.
+# This dict should be only used in json parsing.
+# Note that:
+# 1, CharType and VarcharType are not listed here, since they need regex;
+# 2, DecimalType can be parsed by both mapping ('decimal') and regex
('decimal(10, 2)');
+# 3, CalendarIntervalType is not an atomic type, but can be mapped by
'interval';
+_all_mappable_types: Dict[str, Type[DataType]] = {
+ "string": StringType,
Review Comment:
There is another issue: existing `typeName` are not consistent with the
Scala side.
I want to fix it in a separate PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
