Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/18444#discussion_r127420412
--- Diff: python/pyspark/sql/tests.py ---
@@ -2312,6 +2317,67 @@ def test_BinaryType_serialization(self):
df = self.spark.createDataFrame(data, schema=schema)
df.collect()
+ # test for SPARK-16542
+ def test_array_types(self):
--- End diff --
Minor suggestions:
```diff
@@ -2327,23 +2327,29 @@ class SQLTests(ReusedPySparkTestCase):
# See: https://docs.python.org/2/library/array.html
def assertCollectSuccess(typecode, value):
- a = array.array(typecode, [value])
- row = Row(myarray=a)
+ row = Row(myarray=array.array(typecode, [value]))
df = self.spark.createDataFrame([row])
- self.assertEqual(df.collect()[0]["myarray"][0], value)
+ self.assertEqual(df.first()["myarray"][0], value)
- supported_types = []
-
- # test string types
+ # supported string types
+ #
+ # blabla... "u" will be removed in python 4 blabla...
+ # and "c" not supported in python 3 blabla ...
+ supported_string_types = []
if sys.version_info[0] < 4:
- supported_types += ['u']
+ supported_string_types += ['u']
+ # test unicode
assertCollectSuccess('u', u"a")
if sys.version_info[0] < 3:
- supported_types += ['c']
+ supported_string_types += ['c']
+ # test string
assertCollectSuccess('c', "a")
+ # supported float and double
+ #
+ # tests float max min blabla
+ supported_fractional_types = ['f', 'd']
# test float and double, assuming IEEE 754 floating-point format
- supported_types += ['f', 'd']
assertCollectSuccess('f', ctypes.c_float(1e+38).value)
assertCollectSuccess('f', ctypes.c_float(1e-38).value)
assertCollectSuccess('f', ctypes.c_float(1.123456).value)
@@ -2351,33 +2357,48 @@ class SQLTests(ReusedPySparkTestCase):
assertCollectSuccess('d', sys.float_info.min)
assertCollectSuccess('d', sys.float_info.epsilon)
+ # supported int types
+ #
+ # blabla .. only supported int types.. blabla..
+ supported_int_types = list(
+ set(_array_int_typecode_ctype_mappings.keys())
+ .intersection(set(_array_type_mappings.keys())))
# test int types
- supported_int =
list(set(_array_int_typecode_ctype_mappings.keys()).
-
intersection(set(_array_type_mappings.keys())))
- supported_types += supported_int
- for i in supported_int:
- ctype = _array_int_typecode_ctype_mappings[i]
- if i.isupper(): # unsigned
- assertCollectSuccess(i, 2 ** (ctypes.sizeof(ctype) * 8) -
1)
- else: # signed
+ for t in supported_int_types:
+ ctype = _array_int_typecode_ctype_mappings[t]
+ if t.isupper():
+ # test unsigned int types
+ assertCollectSuccess(t, 2 ** (ctypes.sizeof(ctype) * 8) -
1)
+ else:
+ # test signed int types
max_val = 2 ** (ctypes.sizeof(ctype) * 8 - 1)
- assertCollectSuccess(i, max_val - 1)
- assertCollectSuccess(i, -max_val)
-
- # make sure that the test case cover all supported types
+ assertCollectSuccess(t, max_val - 1)
+ assertCollectSuccess(t, -max_val)
+
+ # all supported types
+ #
+ # Make sure all the supported types are blabla ...
+ supported_types = (supported_string_types +
+ supported_fractional_types +
+ supported_int_types)
+ # test these are all supported types
self.assertEqual(set(supported_types),
set(_array_type_mappings.keys()))
- # test unsupported types
+ # all unsupported types
+ #
+ # ... ... types are not supported in python 2/3 blabla.
if sys.version_info[0] < 3:
- all_type_codes = set(['c', 'b', 'B', 'u', 'h', 'H', 'i', 'I',
'l', 'L', 'f', 'd'])
+ all_types = set(['c', 'b', 'B', 'u', 'h', 'H', 'i', 'I', 'l',
'L', 'f', 'd'])
else:
- all_type_codes = set(array.typecodes)
- unsupported_types = all_type_codes - set(supported_types)
+ all_types = set(array.typecodes)
+ unsupported_types = all_types - set(supported_types)
+ # test unsupported types
for t in unsupported_types:
with self.assertRaises(TypeError):
a = array.array(t)
self.spark.createDataFrame([Row(myarray=a)]).collect()
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
