HeartSaVioR commented on issue #26173: [SPARK-29503][SQL] Copy result row from RowWriter in GenerateUnsafeProjection when the expression is lambdaFunction in MapObject URL: https://github.com/apache/spark/pull/26173#issuecomment-544715361 Looks like escaping JSON doesn't seem to work consistently during compiling generated code. `val typeToJson = StringEscapeUtils.escapeJava(StringEscapeUtils.escapeJson(dataType.json))` > DataFrameComplexTypeSuite.SPARK-29503 nest unsafe struct inside safe array ``` arrayData_0.copyUnsafeData("{\\\"type\\\":\\\"array\\\",\\\"elementType\\\":{\\\"type\\\":\\\"struct\\\",\\\"fields\\\":[{\\\"name\\\":\\\"col1\\\",\\\"type\\\":\\\"integer\\\",\\\"nullable\\\":true,\\\"metadata\\\":{}}]},\\\"containsNull\\\":false}") ``` ``` /* 140 */ if (false) { /* 141 */ convertedArray_0[loopIndex_0] = null; /* 142 */ } else { /* 143 */ convertedArray_0[loopIndex_0] = arrayData_0.copyUnsafeData("{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":false}"); /* 144 */ } ``` > ALSSuite.recommendForAllUsers with k <, = and > num_items ``` serializefromobject_value_2.copyUnsafeData("{\\\"type\\\":\\\"struct\\\",\\\"fields\\\":[{\\\"name\\\":\\\"_1\\\",\\\"type\\\":\\\"integer\\\",\\\"nullable\\\":false,\\\"metadata\\\":{}},{\\\"name\\\":\\\"_2\\\",\\\"type\\\":{\\\"type\\\":\\\"array\\\",\\\"elementType\\\":\\\"float\\\",\\\"containsNull\\\":false},\\\"nullable\\\":true,\\\"metadata\\\":{}}]}") ``` ``` /* 112 */ if (serializefromobject_isNull_2) { /* 113 */ serializefromobject_convertedArray_0[serializefromobject_loopIndex_0] = null; /* 114 */ } else { /* 115 */ serializefromobject_convertedArray_0[serializefromobject_loopIndex_0] = serializefromobject_value_2.copyUnsafeData("{"type":"struct","fields":[{"name":"_1","type":"integer","nullable":false,"metadata":{}},{"name":"_2","type":{"type":"array","elementType":"float","containsNull":false},"nullable":true,"metadata":{}}]}"); /* 116 */ } ``` ---- `val typeToJson = StringEscapeUtils.escapeJson(dataType.json)` > DataFrameComplexTypeSuite.SPARK-29503 nest unsafe struct inside safe array ``` arrayData_0.copyUnsafeData("{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":false}") ``` ``` /* 140 */ if (false) { /* 141 */ convertedArray_0[loopIndex_0] = null; /* 142 */ } else { /* 143 */ convertedArray_0[loopIndex_0] = arrayData_0.copyUnsafeData("{"type":"array","elementType":{"type":"struct","fields":[{"name":"col1","type":"integer","nullable":true,"metadata":{}}]},"containsNull":false}"); /* 144 */ } ``` > ALSSuite.recommendForAllUsers with k <, = and > num_items ``` serializefromobject_value_2.copyUnsafeData("{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"integer\",\"nullable\":false,\"metadata\":{}},{\"name\":\"_2\",\"type\":{\"type\":\"array\",\"elementType\":\"float\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}") ``` ``` /* 112 */ if (serializefromobject_isNull_2) { /* 113 */ serializefromobject_convertedArray_0[serializefromobject_loopIndex_0] = null; /* 114 */ } else { /* 115 */ serializefromobject_convertedArray_0[serializefromobject_loopIndex_0] = serializefromobject_value_2.copyUnsafeData("{"type":"struct","fields":[{"name":"_1","type":"integer","nullable":false,"metadata":{}},{"name":"_2","type":{"type":"array","elementType":"float","containsNull":false},"nullable":true,"metadata":{}}]}"); /* 116 */ } ``` ---- `val typeToJson = StringEscapeUtils.escapeJava(dataType.json)` > DataFrameComplexTypeSuite.SPARK-29503 nest unsafe struct inside safe array ``` arrayData_0.copyUnsafeData("{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":false}") ``` ``` /* 140 */ if (false) { /* 141 */ convertedArray_0[loopIndex_0] = null; /* 142 */ } else { /* 143 */ convertedArray_0[loopIndex_0] = arrayData_0.copyUnsafeData("{"type":"array","elementType":{"type":"struct","fields":[{"name":"col1","type":"integer","nullable":true,"metadata":{}}]},"containsNull":false}"); /* 144 */ } ``` > ALSSuite.recommendForAllUsers with k <, = and > num_items ``` serializefromobject_value_2.copyUnsafeData("{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"integer\",\"nullable\":false,\"metadata\":{}},{\"name\":\"_2\",\"type\":{\"type\":\"array\",\"elementType\":\"float\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}") ``` ``` /* 112 */ if (serializefromobject_isNull_2) { /* 113 */ serializefromobject_convertedArray_0[serializefromobject_loopIndex_0] = null; /* 114 */ } else { /* 115 */ serializefromobject_convertedArray_0[serializefromobject_loopIndex_0] = serializefromobject_value_2.copyUnsafeData("{"type":"struct","fields":[{"name":"_1","type":"integer","nullable":false,"metadata":{}},{"name":"_2","type":{"type":"array","elementType":"float","containsNull":false},"nullable":true,"metadata":{}}]}"); /* 116 */ } ``` So double escaping seems to be only working solution for new UT, but consistently fails on ALSSuite - `\\` escape is gone. Not sure why.
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
