[
https://issues.apache.org/jira/browse/SPARK-9675?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14659342#comment-14659342
]
Reynold Xin commented on SPARK-9675:
------------------------------------
An example of corrupted data. See the map part. 1st line is the golden answer,
and 2nd line is what gets output from Exchange with GenerateUnsafeProjection.
{code}
[str100: test cache.,WrappedArray(98, 105, 110, 97, 114, 121, 49, 48, 48, 58,
32, 116, 101, 115, 116, 32, 99, 97, 99, 104, 101,
46),null,true,100,100,100,9223372036854775707,100.25,100.75,9223372036854775807.12345,2.23456,1969-12-31,1969-12-31
16:00:00.1,WrappedArray(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56,
57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76,
77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96,
97, 98, 99, 100),Map(map_key_71 -> 9223372036854775736, map_key_75 ->
9223372036854775732, map_key_35 -> 9223372036854775772, map_key_20 ->
9223372036854775787, map_key_60 -> 9223372036854775747, map_key_29 ->
9223372036854775778, map_key_31 -> 9223372036854775776, map_key_3 ->
9223372036854775804, map_key_83 -> 9223372036854775724, map_key_67 ->
9223372036854775740, map_key_7 -> 9223372036854775800, map_key_13 ->
9223372036854775794, map_key_49 -> 9223372036854775758, map_key_94 ->
9223372036854775713, map_key_56 -> 9223372036854775751, map_key_97 ->
9223372036854775710, map_key_72 -> 9223372036854775735, map_key_21 ->
9223372036854775786, map_key_86 -> 9223372036854775721, map_key_45 ->
9223372036854775762, map_key_79 -> 9223372036854775728, map_key_18 ->
9223372036854775789, map_key_24 -> 9223372036854775783, map_key_61 ->
9223372036854775746, map_key_36 -> 9223372036854775771, map_key_91 ->
9223372036854775716, map_key_44 -> 9223372036854775763, map_key_30 ->
9223372036854775777, map_key_84 -> 9223372036854775723, map_key_78 ->
9223372036854775729, map_key_32 -> 9223372036854775775, map_key_50 ->
9223372036854775757, map_key_10 -> 9223372036854775797, map_key_80 ->
9223372036854775727, map_key_66 -> 9223372036854775741, map_key_25 ->
9223372036854775782, map_key_14 -> 9223372036854775793, map_key_59 ->
9223372036854775748, map_key_89 -> 9223372036854775718, map_key_28 ->
9223372036854775779, map_key_6 -> 9223372036854775801, map_key_17 ->
9223372036854775790, map_key_39 -> 9223372036854775768, map_key_95 ->
9223372036854775712, map_key_55 -> 9223372036854775752, map_key_0 ->
9223372036854775807, map_key_48 -> 9223372036854775759, map_key_77 ->
9223372036854775730, map_key_37 -> 9223372036854775770, map_key_92 ->
9223372036854775715, map_key_81 -> 9223372036854775726, map_key_70 ->
9223372036854775737, map_key_11 -> 9223372036854775796, map_key_22 ->
9223372036854775785, map_key_62 -> 9223372036854775745, map_key_51 ->
9223372036854775756, map_key_40 -> 9223372036854775767, map_key_73 ->
9223372036854775734, map_key_90 -> 9223372036854775717, map_key_69 ->
9223372036854775738, map_key_9 -> 9223372036854775798, map_key_88 ->
9223372036854775719, map_key_43 -> 9223372036854775764, map_key_26 ->
9223372036854775781, map_key_65 -> 9223372036854775742, map_key_99 ->
9223372036854775708, map_key_5 -> 9223372036854775802, map_key_54 ->
9223372036854775753, map_key_47 -> 9223372036854775760, map_key_33 ->
9223372036854775774, map_key_96 -> 9223372036854775711, map_key_1 ->
9223372036854775806, map_key_85 -> 9223372036854775722, map_key_58 ->
9223372036854775749, map_key_15 -> 9223372036854775792, map_key_12 ->
9223372036854775795, map_key_98 -> 9223372036854775709, map_key_52 ->
9223372036854775755, map_key_38 -> 9223372036854775769, map_key_76 ->
9223372036854775731, map_key_23 -> 9223372036854775784, map_key_2 ->
9223372036854775805, map_key_41 -> 9223372036854775766, map_key_63 ->
9223372036854775744, map_key_8 -> 9223372036854775799, map_key_74 ->
9223372036854775733, map_key_27 -> 9223372036854775780, map_key_93 ->
9223372036854775714, map_key_53 -> 9223372036854775754, map_key_42 ->
9223372036854775765, map_key_82 -> 9223372036854775725, map_key_68 ->
9223372036854775739, map_key_4 -> 9223372036854775803, map_key_19 ->
9223372036854775788, map_key_57 -> 9223372036854775750, map_key_16 ->
9223372036854775791, map_key_46 -> 9223372036854775761, map_key_87 ->
9223372036854775720, map_key_64 -> 9223372036854775743, map_key_100 ->
9223372036854775707, map_key_34 ->
9223372036854775773),[99.75,WrappedArray(true, false, null)]]
[str100: test cache.,WrappedArray(98, 105, 110, 97, 114, 121, 49, 48, 48, 58,
32, 116, 101, 115, 116, 32, 99, 97, 99, 104, 101,
46),null,true,100,100,100,9223372036854775707,100.25,100.75,9223372036854775807.12345,2.23456,1969-12-31,1969-12-31
16:00:00.1,WrappedArray(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56,
57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76,
77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96,
97, 98, 99, 100),Map(p_key_27ma -> 9223372036854775733, p_key_4map ->
9223372036854775739, p_key_70ma -> 9223372036854775726, p_key_30ma ->
9223372036854775763, p_key_62ma -> 9223372036854775785, p_key_44ma ->
9223372036854775716, p_key_52ma -> 9223372036854775709, p_key_73ma ->
9223372036854775767, p_key_21ma -> 9223372036854775735, p_key_45ma ->
9223372036854775721, p_key_58ma -> 9223372036854775722, p_key_50ma ->
9223372036854775775, p_key_86ma -> 9223372036854775786, p_key_67ma ->
9223372036854775724, _key_17ma -> 9223372036854775801, p_key_40ma ->
9223372036854775756, p_key_12ma -> 9223372036854775792, p_key_31ma ->
9223372036854775778, p_key_57ma -> 9223372036854775788, p_key_26ma ->
9223372036854775764, p_key_65ma -> 9223372036854775781, p_key_24ma ->
9223372036854775789, p_key_53ma -> 9223372036854775714, p_key_94ma ->
9223372036854775758, p_key_56ma -> 9223372036854775713, p_key_79ma ->
9223372036854775762, _key_48ma -> 9223372036854775807, p_key_76ma ->
9223372036854775769, ??? -> 9223372036854775707, p_key_90ma ->
9223372036854775734, p_key_11ma -> 9223372036854775737, p_key_37ma ->
9223372036854775730, p_key_14ma -> 9223372036854775782, p_key_20ma ->
9223372036854775772, p_key_47ma -> 9223372036854775753, p_key_69ma ->
9223372036854775717, _key_85ma -> 9223372036854775806, p_key_59ma ->
9223372036854775793, p_key_46ma -> 9223372036854775791, p_key_80ma ->
9223372036854775797, p_key_93ma -> 9223372036854775780, p_key_33ma ->
9223372036854775760, p_key_63ma -> 9223372036854775766, p_key_9map ->
9223372036854775738, p_key_66ma -> 9223372036854775727, p_key_87ma ->
9223372036854775761, _key_74ma -> 9223372036854775799, p_key_81ma ->
9223372036854775715, p_key_0map -> 9223372036854775752, p_key_99ma ->
9223372036854775742, p_key_29ma -> 9223372036854775747, p_key_5map ->
9223372036854775708, p_key_7map -> 9223372036854775740, _key_88ma ->
9223372036854775798, p_key_72ma -> 9223372036854775710, p_key_35ma ->
9223372036854775732, p_key_92ma -> 9223372036854775770, p_key_96ma ->
9223372036854775774, p_key_49ma -> 9223372036854775794, p_key_84ma ->
9223372036854775777, p_key_10ma -> 9223372036854775757, p_key_38ma ->
9223372036854775755, p_key_60ma -> 9223372036854775787, p_key_32ma ->
9223372036854775729, p_key_15ma -> 9223372036854775749, p_key_16ma ->
9223372036854775750, p_key_97ma -> 9223372036854775751, p_key_42ma ->
9223372036854775754, p_key_23ma -> 9223372036854775731, p_key_8map ->
9223372036854775744, _key_83ma -> 9223372036854775804, p_key_75ma ->
9223372036854775736, p_key_1map -> 9223372036854775711, p_key_25ma ->
9223372036854775741, _key_13ma -> 9223372036854775800, p_key_39ma ->
9223372036854775790, _key_19ma -> 9223372036854775803, p_key_98ma ->
9223372036854775795, p_key_51ma -> 9223372036854775745, p_key_78ma ->
9223372036854775723, p_key_89ma -> 9223372036854775748, p_key_82ma ->
9223372036854775765, p_key_18ma -> 9223372036854775728, _key_41ma ->
9223372036854775805, p_key_55ma -> 9223372036854775712, p_key_28ma ->
9223372036854775718, p_key_6map -> 9223372036854775779, p_key_43ma ->
9223372036854775719, p_key_34? -> 9223372036854775743, p_key_36ma ->
9223372036854775746, p_key_61ma -> 9223372036854775783, p_key_77ma ->
9223372036854775759, p_key_68ma -> 9223372036854775725, p_key_64ma ->
9223372036854775720, ?? -> 9223372036854775773, p_key_2map ->
9223372036854775784, p_key_3map -> 9223372036854775776, _key_54ma ->
9223372036854775802, p_key_91ma -> 9223372036854775771, p_key_22ma ->
9223372036854775796, p_key_95ma ->
9223372036854775768),[99.75,WrappedArray(true, false, null)]]
{code}
> GenerateUnsafeProjection seems to corrupt MapType data
> ------------------------------------------------------
>
> Key: SPARK-9675
> URL: https://issues.apache.org/jira/browse/SPARK-9675
> Project: Spark
> Issue Type: Improvement
> Components: SQL
> Reporter: Reynold Xin
> Assignee: Wenchen Fan
> Priority: Blocker
>
> See https://github.com/apache/spark/pull/7981#issuecomment-128208233
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
