Github user JoshRosen commented on a diff in the pull request:
https://github.com/apache/spark/pull/2365#discussion_r17509580
--- Diff: core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
---
@@ -28,6 +30,56 @@ import org.apache.spark.rdd.RDD
/** Utilities for serialization / deserialization between Python and Java,
using Pickle. */
private[python] object SerDeUtil extends Logging {
+ // Unpickle array.array generated by Python 2.6
+ class ArrayConstructor extends
net.razorvine.pickle.objects.ArrayConstructor {
+ // /* Description of types */
+ // static struct arraydescr descriptors[] = {
+ // {'c', sizeof(char), c_getitem, c_setitem},
+ // {'b', sizeof(char), b_getitem, b_setitem},
+ // {'B', sizeof(char), BB_getitem, BB_setitem},
+ // #ifdef Py_USING_UNICODE
+ // {'u', sizeof(Py_UNICODE), u_getitem, u_setitem},
+ // #endif
+ // {'h', sizeof(short), h_getitem, h_setitem},
+ // {'H', sizeof(short), HH_getitem, HH_setitem},
+ // {'i', sizeof(int), i_getitem, i_setitem},
+ // {'I', sizeof(int), II_getitem, II_setitem},
+ // {'l', sizeof(long), l_getitem, l_setitem},
+ // {'L', sizeof(long), LL_getitem, LL_setitem},
+ // {'f', sizeof(float), f_getitem, f_setitem},
+ // {'d', sizeof(double), d_getitem, d_setitem},
+ // {'\0', 0, 0, 0} /* Sentinel */
+ // };
+ // TODO: support Py_UNICODE with 2 bytes
+ // FIXME: unpickle array of float is wrong in Pyrolite, so we reverse
the
+ // machine code for float/double here to work arround it.
+ // we should fix this after Pyrolite fix them
+ val machineCodes: Map[Char, Int] = if
(ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
+ Map('c' -> 1, 'B' -> 0, 'b' -> 1, 'H' -> 3, 'h' -> 5, 'I' -> 7, 'i'
-> 9,
+ 'L' -> 11, 'l' -> 13, 'f' -> 14, 'd' -> 16, 'u' -> 21
+ )
+ } else {
+ Map('c' -> 1, 'B' -> 0, 'b' -> 1, 'H' -> 2, 'h' -> 4, 'I' -> 6, 'i'
-> 8,
+ 'L' -> 10, 'l' -> 12, 'f' -> 15, 'd' -> 17, 'u' -> 20
+ )
+ }
+ override def construct(args: Array[Object]): Object = {
+ if (args.length == 1) {
+ construct(args ++ Array(""))
+ } else if (args.length == 2 && args(1).isInstanceOf[String]) {
+ val typecode = args(0).asInstanceOf[String].charAt(0)
+ val data: String = args(1).asInstanceOf[String]
+ println(typecode, machineCodes(typecode), data.length, data.toList)
--- End diff --
Can you remove this debugging statement?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
