LuciferYang commented on code in PR #40796:
URL: https://github.com/apache/spark/pull/40796#discussion_r1202145924
##########
connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala:
##########
@@ -197,4 +197,22 @@ class UserDefinedFunctionE2ETestSuite extends
RemoteSparkSession {
spark.range(10).repartition(1).foreachPartition(func)
assert(sum.get() == 0) // The value is not 45
}
+
+ test("Dataset reduce") {
Review Comment:
```java
23/05/23 13:40:31 WARN GenerateUnsafeProjection: code for
assertnotnull(input[0, scala.Tuple2, true])._1,assertnotnull(input[0,
scala.Tuple2, true])._2:
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends
org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private boolean subExprIsNull_0;
/* 009 */ private
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[]
mutableStateArray_1 = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1];
/* 010 */ private scala.Tuple2[] mutableStateArray_0 = new scala.Tuple2[1];
/* 011 */
/* 012 */ public SpecificUnsafeProjection(Object[] references) {
/* 013 */ this.references = references;
/* 014 */
/* 015 */ mutableStateArray_1[0] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0);
/* 016 */
/* 017 */ }
/* 018 */
/* 019 */ public void initialize(int partitionIndex) {
/* 020 */
/* 021 */ }
/* 022 */
/* 023 */ // Scala.Function1 need this
/* 024 */ public java.lang.Object apply(java.lang.Object row) {
/* 025 */ return apply((InternalRow) row);
/* 026 */ }
/* 027 */
/* 028 */ public UnsafeRow apply(InternalRow i) {
/* 029 */ mutableStateArray_1[0].reset();
/* 030 */ subExpr_0(i);
/* 031 */
/* 032 */ mutableStateArray_1[0].zeroOutNullBytes();
/* 033 */
/* 034 */ boolean isNull_2 = true;
/* 035 */ boolean value_2 = false;
/* 036 */ isNull_2 = false;
/* 037 */ if (!isNull_2) {
/* 038 */
/* 039 */ Object funcResult_0 = null;
/* 040 */ funcResult_0 = mutableStateArray_0[0]._1();
/* 041 */ value_2 = (Boolean) funcResult_0;
/* 042 */
/* 043 */ }
/* 044 */ mutableStateArray_1[0].write(0, value_2);
/* 045 */
/* 046 */ boolean isNull_3 = true;
/* 047 */ long value_3 = -1L;
/* 048 */ isNull_3 = false;
/* 049 */ if (!isNull_3) {
/* 050 */
/* 051 */ Object funcResult_1 = null;
/* 052 */ funcResult_1 = mutableStateArray_0[0]._2();
/* 053 */ value_3 = (Long) funcResult_1;
/* 054 */
/* 055 */ }
/* 056 */ mutableStateArray_1[0].write(1, value_3);
/* 057 */ return (mutableStateArray_1[0].getRow());
/* 058 */ }
/* 059 */
/* 060 */
/* 061 */ private void subExpr_0(InternalRow i) {
/* 062 */ boolean isNull_1 = i.isNullAt(0);
/* 063 */ scala.Tuple2 value_1 = isNull_1 ?
/* 064 */ null : ((scala.Tuple2)i.get(0, null));
/* 065 */ if (isNull_1) {
/* 066 */ throw new NullPointerException(((java.lang.String)
references[0] /* errMsg */));
/* 067 */ }
/* 068 */ subExprIsNull_0 = false;
/* 069 */ mutableStateArray_0[0] = value_1;
/* 070 */ }
/* 071 */
/* 072 */ }
```
NPE should be thrown from line 53 of generated code, and the input data
should be `(false, null)` (ReduceAggregator.zero).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
