Suresh Avadhanula created SPARK-19418: -----------------------------------------
Summary: Dataset generated java code fails to compile as java.lang.Long does not accept UTF8String in constructor Key: SPARK-19418 URL: https://issues.apache.org/jira/browse/SPARK-19418 Project: Spark Issue Type: Bug Components: SQL Affects Versions: 2.1.0 Reporter: Suresh Avadhanula I have the following in Java spark driver. DealerPerson module object is {code:title=DealerPerson.java|borderStyle=solid} public class DealerPerson { Long schemaOrgUnitId ; List<Person> personList } {code} I populate it using group by as follows. {code} Dataset<DealerPerson> dps = persondds.groupByKey(new MapFunction<Person, Long>() { @Override public Long call(Person person) throws Exception { return person.getSchemaOrgUnitId(); } }, Encoders.LONG()). mapGroups(new MapGroupsFunction<Long, Person, DealerPerson>() { @Override public DealerPerson call(Long dp, java.util.Iterator<Person> iterator) throws Exception { DealerPerson retDp = new DealerPerson(); retDp.setSchemaOrgUnitId(dp); ArrayList<Person> persons = new ArrayList<Person>(); while (iterator.hasNext()) persons.add(iterator.next()); retDp.setPersons(persons); return retDp; } }, Encoders.bean(DealerPerson.class)); {code} The generated code throws compiler exception since UTF8String is java.lang.Long() {noformat} 7/01/31 20:32:28 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 5, localhost, executor driver, partition 0, PROCESS_LOCAL, 6442 bytes) 17/01/31 20:32:28 INFO Executor: Running task 0.0 in stage 5.0 (TID 5) 17/01/31 20:32:28 ERROR CodeGenerator: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 56, Column 58: No applicable constructor/method found for actual parameters "org.apache.spark.unsafe.types.UTF8String"; candidates are: "java.lang.Long(long)", "java.lang.Long(java.lang.String)" /* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificSafeProjection(references); /* 003 */ } /* 004 */ /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { /* 006 */ /* 007 */ private Object[] references; /* 008 */ private InternalRow mutableRow; /* 009 */ private com.xtime.spark.model.Person javaBean; /* 010 */ private boolean resultIsNull; /* 011 */ private UTF8String argValue; /* 012 */ private boolean resultIsNull1; /* 013 */ private UTF8String argValue1; /* 014 */ private boolean resultIsNull2; /* 015 */ private UTF8String argValue2; /* 016 */ private boolean resultIsNull3; /* 017 */ private UTF8String argValue3; /* 018 */ private boolean resultIsNull4; /* 019 */ private UTF8String argValue4; /* 020 */ /* 021 */ public SpecificSafeProjection(Object[] references) { /* 022 */ this.references = references; /* 023 */ mutableRow = (InternalRow) references[references.length - 1]; /* 024 */ /* 025 */ /* 026 */ /* 027 */ /* 028 */ /* 029 */ /* 030 */ /* 031 */ /* 032 */ /* 033 */ /* 034 */ /* 035 */ /* 036 */ } /* 037 */ /* 038 */ public void initialize(int partitionIndex) { /* 039 */ /* 040 */ } /* 041 */ /* 042 */ /* 043 */ private void apply_4(InternalRow i) { /* 044 */ /* 045 */ /* 046 */ resultIsNull1 = false; /* 047 */ if (!resultIsNull1) { /* 048 */ /* 049 */ boolean isNull21 = i.isNullAt(17); /* 050 */ UTF8String value21 = isNull21 ? null : (i.getUTF8String(17)); /* 051 */ resultIsNull1 = isNull21; /* 052 */ argValue1 = value21; /* 053 */ } /* 054 */ /* 055 */ /* 056 */ final java.lang.Long value20 = resultIsNull1 ? null : new java.lang.Long(argValue1); /* 057 */ javaBean.setSchemaOrgUnitId(value20); /* 058 */ /* 059 */ /* 060 */ resultIsNull2 = false; /* 061 */ if (!resultIsNull2) { /* 062 */ /* 063 */ boolean isNull23 = i.isNullAt(0); /* 064 */ UTF8String value23 = isNull23 ? null : (i.getUTF8String(0)); /* 065 */ resultIsNull2 = isNull23; /* 066 */ argValue2 = value23; /* 067 */ } /* 068 */ /* 069 */ /* 070 */ final java.lang.Long value22 = resultIsNull2 ? null : new java.lang.Long(argValue2); /* 071 */ javaBean.setPersonId(value22); /* 072 */ /* 073 */ /* 074 */ boolean isNull25 = i.isNullAt(10); /* 075 */ UTF8String value25 = isNull25 ? null : (i.getUTF8String(10)); /* 076 */ boolean isNull24 = true; /* 077 */ java.lang.String value24 = null; /* 078 */ if (!isNull25) { /* 079 */ /* 080 */ isNull24 = false; /* 081 */ if (!isNull24) { /* 082 */ /* 083 */ Object funcResult8 = null; /* 084 */ funcResult8 = value25.toString(); /* 085 */ if (funcResult8 == null) { /* 086 */ isNull24 = true; /* 087 */ } else { /* 088 */ value24 = (java.lang.String) funcResult8; /* 089 */ } /* 090 */ /* 091 */ } /* 092 */ isNull24 = value24 == null; /* 093 */ } /* 094 */ javaBean.setCreateMethod(value24); /* 095 */ /* 096 */ } /* 097 */ /* 098 */ /* {noformat} -- This message was sent by Atlassian JIRA (v6.3.15#6346) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org