Dinesh Man Amatya created SPARK-20176:
-----------------------------------------
Summary: Spark Dataframe UDAF issue
Key: SPARK-20176
URL: https://issues.apache.org/jira/browse/SPARK-20176
Project: Spark
Issue Type: IT Help
Components: Spark Core
Affects Versions: 2.0.2
Reporter: Dinesh Man Amatya
Getting following error in custom UDAF
Error while decoding: java.util.concurrent.ExecutionException:
java.lang.Exception: failed to compile:
org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 58,
Column 33: Incompatible expression types "boolean" and "java.lang.Boolean"
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificSafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificSafeProjection extends
org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private MutableRow mutableRow;
/* 009 */ private Object[] values;
/* 010 */ private Object[] values1;
/* 011 */ private org.apache.spark.sql.types.StructType schema;
/* 012 */ private org.apache.spark.sql.types.StructType schema1;
/* 013 */
/* 014 */
/* 015 */ public SpecificSafeProjection(Object[] references) {
/* 016 */ this.references = references;
/* 017 */ mutableRow = (MutableRow) references[references.length - 1];
/* 018 */
/* 019 */
/* 020 */ this.schema = (org.apache.spark.sql.types.StructType)
references[0];
/* 021 */ this.schema1 = (org.apache.spark.sql.types.StructType)
references[1];
/* 022 */ }
/* 023 */
/* 024 */ public java.lang.Object apply(java.lang.Object _i) {
/* 025 */ InternalRow i = (InternalRow) _i;
/* 026 */
/* 027 */ values = new Object[2];
/* 028 */
/* 029 */ boolean isNull2 = i.isNullAt(0);
/* 030 */ UTF8String value2 = isNull2 ? null : (i.getUTF8String(0));
/* 031 */
/* 032 */ boolean isNull1 = isNull2;
/* 033 */ final java.lang.String value1 = isNull1 ? null :
(java.lang.String) value2.toString();
/* 034 */ isNull1 = value1 == null;
/* 035 */ if (isNull1) {
/* 036 */ values[0] = null;
/* 037 */ } else {
/* 038 */ values[0] = value1;
/* 039 */ }
/* 040 */
/* 041 */ boolean isNull5 = i.isNullAt(1);
/* 042 */ InternalRow value5 = isNull5 ? null : (i.getStruct(1, 2));
/* 043 */ boolean isNull3 = false;
/* 044 */ org.apache.spark.sql.Row value3 = null;
/* 045 */ if (!false && isNull5) {
/* 046 */
/* 047 */ final org.apache.spark.sql.Row value6 = null;
/* 048 */ isNull3 = true;
/* 049 */ value3 = value6;
/* 050 */ } else {
/* 051 */
/* 052 */ values1 = new Object[2];
/* 053 */
/* 054 */ boolean isNull10 = i.isNullAt(1);
/* 055 */ InternalRow value10 = isNull10 ? null : (i.getStruct(1, 2));
/* 056 */
/* 057 */ boolean isNull9 = isNull10 || false;
/* 058 */ final boolean value9 = isNull9 ? false : (Boolean)
value10.isNullAt(0);
/* 059 */ boolean isNull8 = false;
/* 060 */ double value8 = -1.0;
/* 061 */ if (!isNull9 && value9) {
/* 062 */
/* 063 */ final double value12 = -1.0;
/* 064 */ isNull8 = true;
/* 065 */ value8 = value12;
/* 066 */ } else {
/* 067 */
/* 068 */ boolean isNull14 = i.isNullAt(1);
/* 069 */ InternalRow value14 = isNull14 ? null : (i.getStruct(1, 2));
/* 070 */ boolean isNull13 = isNull14;
/* 071 */ double value13 = -1.0;
/* 072 */
/* 073 */ if (!isNull14) {
/* 074 */
/* 075 */ if (value14.isNullAt(0)) {
/* 076 */ isNull13 = true;
/* 077 */ } else {
/* 078 */ value13 = value14.getDouble(0);
/* 079 */ }
/* 080 */
/* 081 */ }
/* 082 */ isNull8 = isNull13;
/* 083 */ value8 = value13;
/* 084 */ }
/* 085 */ if (isNull8) {
/* 086 */ values1[0] = null;
/* 087 */ } else {
/* 088 */ values1[0] = value8;
/* 089 */ }
/* 090 */
/* 091 */ boolean isNull17 = i.isNullAt(1);
/* 092 */ InternalRow value17 = isNull17 ? null : (i.getStruct(1, 2));
/* 093 */
/* 094 */ boolean isNull16 = isNull17 || false;
/* 095 */ final boolean value16 = isNull16 ? false : (Boolean)
value17.isNullAt(1);
/* 096 */ boolean isNull15 = false;
/* 097 */ double value15 = -1.0;
/* 098 */ if (!isNull16 && value16) {
/* 099 */
/* 100 */ final double value19 = -1.0;
/* 101 */ isNull15 = true;
/* 102 */ value15 = value19;
/* 103 */ } else {
/* 104 */
/* 105 */ boolean isNull21 = i.isNullAt(1);
/* 106 */ InternalRow value21 = isNull21 ? null : (i.getStruct(1, 2));
/* 107 */ boolean isNull20 = isNull21;
/* 108 */ double value20 = -1.0;
/* 109 */
/* 110 */ if (!isNull21) {
/* 111 */
/* 112 */ if (value21.isNullAt(1)) {
/* 113 */ isNull20 = true;
/* 114 */ } else {
/* 115 */ value20 = value21.getDouble(1);
/* 116 */ }
/* 117 */
/* 118 */ }
/* 119 */ isNull15 = isNull20;
/* 120 */ value15 = value20;
/* 121 */ }
/* 122 */ if (isNull15) {
/* 123 */ values1[1] = null;
/* 124 */ } else {
/* 125 */ values1[1] = value15;
/* 126 */ }
/* 127 */
/* 128 */ final org.apache.spark.sql.Row value7 = new
org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values1, schema);
/* 129 */ isNull3 = false;
/* 130 */ value3 = value7;
/* 131 */ }
/* 132 */ if (isNull3) {
/* 133 */ values[1] = null;
/* 134 */ } else {
/* 135 */ values[1] = value3;
/* 136 */ }
/* 137 */
/* 138 */ final org.apache.spark.sql.Row value = new
org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values, schema1);
/* 139 */ if (false) {
/* 140 */ mutableRow.setNullAt(0);
/* 141 */ } else {
/* 142 */
/* 143 */ mutableRow.update(0, value);
/* 144 */ }
/* 145 */
/* 146 */ return mutableRow;
/* 147 */ }
/* 148 */ }
createexternalrow(input[0, string, true].toString, if (isnull(input[1,
struct<totalRxPaid:double,totalRxAllowedAmt:double>, true])) null else
createexternalrow(if (input[1,
struct<totalRxPaid:double,totalRxAllowedAmt:double>, true].isNullAt) null else
input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].totalRxPaid, if (input[1,
struct<totalRxPaid:double,totalRxAllowedAmt:double>, true].isNullAt) null else
input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].totalRxAllowedAmt, StructField(totalRxPaid,DoubleType,true),
StructField(totalRxAllowedAmt,DoubleType,true)),
StructField(dw_member_id,StringType,true),
StructField(test,StructType(StructField(totalRxPaid,DoubleType,true),
StructField(totalRxAllowedAmt,DoubleType,true)),true))
:- input[0, string, true].toString
: +- input[0, string, true]
+- if (isnull(input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true])) null else createexternalrow(if (input[1,
struct<totalRxPaid:double,totalRxAllowedAmt:double>, true].isNullAt) null else
input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].totalRxPaid, if (input[1,
struct<totalRxPaid:double,totalRxAllowedAmt:double>, true].isNullAt) null else
input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].totalRxAllowedAmt, StructField(totalRxPaid,DoubleType,true),
StructField(totalRxAllowedAmt,DoubleType,true))
:- isnull(input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true])
: +- input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>, true]
:- null
+- createexternalrow(if (input[1,
struct<totalRxPaid:double,totalRxAllowedAmt:double>, true].isNullAt) null else
input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].totalRxPaid, if (input[1,
struct<totalRxPaid:double,totalRxAllowedAmt:double>, true].isNullAt) null else
input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].totalRxAllowedAmt, StructField(totalRxPaid,DoubleType,true),
StructField(totalRxAllowedAmt,DoubleType,true))
:- if (input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].isNullAt) null else input[1,
struct<totalRxPaid:double,totalRxAllowedAmt:double>, true].totalRxPaid
: :- input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].isNullAt
: : :- input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true]
: : +- 0
: :- null
: +- input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].totalRxPaid
: +- input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true]
+- if (input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].isNullAt) null else input[1,
struct<totalRxPaid:double,totalRxAllowedAmt:double>, true].totalRxAllowedAmt
:- input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].isNullAt
: :- input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true]
: +- 1
:- null
+- input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true].totalRxAllowedAmt
+- input[1, struct<totalRxPaid:double,totalRxAllowedAmt:double>,
true]
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
