Suresh Avadhanula created SPARK-19418:
-----------------------------------------
Summary: Dataset generated java code fails to compile as
java.lang.Long does not accept UTF8String in constructor
Key: SPARK-19418
URL: https://issues.apache.org/jira/browse/SPARK-19418
Project: Spark
Issue Type: Bug
Components: SQL
Affects Versions: 2.1.0
Reporter: Suresh Avadhanula
I have the following in Java spark driver.
DealerPerson module object is
{code:title=DealerPerson.java|borderStyle=solid}
public class DealerPerson
{
Long schemaOrgUnitId ;
List<Person> personList
}
{code}
I populate it using group by as follows.
{code}
Dataset<DealerPerson> dps = persondds.groupByKey(new MapFunction<Person,
Long>() {
@Override
public Long call(Person person) throws Exception {
return person.getSchemaOrgUnitId();
}
}, Encoders.LONG()).
mapGroups(new MapGroupsFunction<Long, Person, DealerPerson>() {
@Override
public DealerPerson call(Long dp,
java.util.Iterator<Person> iterator) throws Exception {
DealerPerson retDp = new DealerPerson();
retDp.setSchemaOrgUnitId(dp);
ArrayList<Person> persons = new ArrayList<Person>();
while (iterator.hasNext())
persons.add(iterator.next());
retDp.setPersons(persons);
return retDp;
}
}, Encoders.bean(DealerPerson.class));
{code}
The generated code throws compiler exception since UTF8String is
java.lang.Long()
{noformat}
7/01/31 20:32:28 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 5,
localhost, executor driver, partition 0, PROCESS_LOCAL, 6442 bytes)
17/01/31 20:32:28 INFO Executor: Running task 0.0 in stage 5.0 (TID 5)
17/01/31 20:32:28 ERROR CodeGenerator: failed to compile:
org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 56,
Column 58: No applicable constructor/method found for actual parameters
"org.apache.spark.unsafe.types.UTF8String"; candidates are:
"java.lang.Long(long)", "java.lang.Long(java.lang.String)"
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificSafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificSafeProjection extends
org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private InternalRow mutableRow;
/* 009 */ private com.xtime.spark.model.Person javaBean;
/* 010 */ private boolean resultIsNull;
/* 011 */ private UTF8String argValue;
/* 012 */ private boolean resultIsNull1;
/* 013 */ private UTF8String argValue1;
/* 014 */ private boolean resultIsNull2;
/* 015 */ private UTF8String argValue2;
/* 016 */ private boolean resultIsNull3;
/* 017 */ private UTF8String argValue3;
/* 018 */ private boolean resultIsNull4;
/* 019 */ private UTF8String argValue4;
/* 020 */
/* 021 */ public SpecificSafeProjection(Object[] references) {
/* 022 */ this.references = references;
/* 023 */ mutableRow = (InternalRow) references[references.length - 1];
/* 024 */
/* 025 */
/* 026 */
/* 027 */
/* 028 */
/* 029 */
/* 030 */
/* 031 */
/* 032 */
/* 033 */
/* 034 */
/* 035 */
/* 036 */ }
/* 037 */
/* 038 */ public void initialize(int partitionIndex) {
/* 039 */
/* 040 */ }
/* 041 */
/* 042 */
/* 043 */ private void apply_4(InternalRow i) {
/* 044 */
/* 045 */
/* 046 */ resultIsNull1 = false;
/* 047 */ if (!resultIsNull1) {
/* 048 */
/* 049 */ boolean isNull21 = i.isNullAt(17);
/* 050 */ UTF8String value21 = isNull21 ? null : (i.getUTF8String(17));
/* 051 */ resultIsNull1 = isNull21;
/* 052 */ argValue1 = value21;
/* 053 */ }
/* 054 */
/* 055 */
/* 056 */ final java.lang.Long value20 = resultIsNull1 ? null : new
java.lang.Long(argValue1);
/* 057 */ javaBean.setSchemaOrgUnitId(value20);
/* 058 */
/* 059 */
/* 060 */ resultIsNull2 = false;
/* 061 */ if (!resultIsNull2) {
/* 062 */
/* 063 */ boolean isNull23 = i.isNullAt(0);
/* 064 */ UTF8String value23 = isNull23 ? null : (i.getUTF8String(0));
/* 065 */ resultIsNull2 = isNull23;
/* 066 */ argValue2 = value23;
/* 067 */ }
/* 068 */
/* 069 */
/* 070 */ final java.lang.Long value22 = resultIsNull2 ? null : new
java.lang.Long(argValue2);
/* 071 */ javaBean.setPersonId(value22);
/* 072 */
/* 073 */
/* 074 */ boolean isNull25 = i.isNullAt(10);
/* 075 */ UTF8String value25 = isNull25 ? null : (i.getUTF8String(10));
/* 076 */ boolean isNull24 = true;
/* 077 */ java.lang.String value24 = null;
/* 078 */ if (!isNull25) {
/* 079 */
/* 080 */ isNull24 = false;
/* 081 */ if (!isNull24) {
/* 082 */
/* 083 */ Object funcResult8 = null;
/* 084 */ funcResult8 = value25.toString();
/* 085 */ if (funcResult8 == null) {
/* 086 */ isNull24 = true;
/* 087 */ } else {
/* 088 */ value24 = (java.lang.String) funcResult8;
/* 089 */ }
/* 090 */
/* 091 */ }
/* 092 */ isNull24 = value24 == null;
/* 093 */ }
/* 094 */ javaBean.setCreateMethod(value24);
/* 095 */
/* 096 */ }
/* 097 */
/* 098 */
/*
{noformat}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
