[
https://issues.apache.org/jira/browse/DRILL-1250?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Rahul Challapalli closed DRILL-1250.
------------------------------------
> org.apache.drill.exec.exception.SchemaChangeException when selecting against
> nested JSON with arrays
> ----------------------------------------------------------------------------------------------------
>
> Key: DRILL-1250
> URL: https://issues.apache.org/jira/browse/DRILL-1250
> Project: Apache Drill
> Issue Type: Bug
> Components: Storage - JSON
> Affects Versions: 0.4.0
> Environment: 5-node hadoop (mapR 3.1.1) cluster
> Reporter: Andy Pernsteiner
> Assignee: Rahul Challapalli
> Fix For: 0.5.0
>
> Attachments: DRILL-1250.1.patch
>
>
> After fighting DRILL-1239, I decided to remove ALL null values from all
> arrays in my nested JSON file. Here's a snippet of the data:
> {code}
> {"trans_id":49995,"date":"07/30/2013","time":"17:29:20","user_info":{"cust_id":6112,"device":"IOS5","state":"tx"},"marketing_info":{"camp_id":7,"searchwords":["sad"]},"trans_info":{"prod_id":[338],"purch_flag":"false"}}
> {"trans_id":49996,"date":"05/18/2013","time":"22:34:58","user_info":{"cust_id":85,"device":"AOS4.2","state":"pa"},"marketing_info":{"camp_id":2,"searchwords":["happy"]},"trans_info":{"prod_id":[143,330,65,12],"purch_flag":"false"}}
> {"trans_id":49997,"date":"08/04/2012","time":"13:19:58","user_info":{"cust_id":14,"device":"AOS4.4","state":"tx"},"marketing_info":{"camp_id":10,"searchwords":["none"]},"trans_info":{"prod_id":[0],"purch_flag":"false"}}
> {"trans_id":49998,"date":"07/19/2013","time":"21:33:09","user_info":{"cust_id":168,"device":"IOS5","state":"tx"},"marketing_info":{"camp_id":7,"searchwords":["sad"]},"trans_info":{"prod_id":[0],"purch_flag":"true"}}
> {"trans_id":49999,"date":"01/29/2013","time":"15:24:21","user_info":{"cust_id":9780,"device":"IOS5","state":"ca"},"marketing_info":{"camp_id":1,"searchwords":["none"]},"trans_info":{"prod_id":[30,149],"purch_flag":"false"}}
> {code}
> If I run the following select query I get the error below:
> select t.trans_id, t.user_info.cust_id as cust_id,
> t.user_info.device as device, t.user_info.state as state,
> t.marketing_info.camp_id as camp_id, t.marketing_info.searchwords,
> t.trans_info.prod_id as prod_id, t.trans_info.purch_flag as purch_flag from
> click.`json_large`.`/nested.json` t;
> {code}
> 0: jdbc:drill:> select t.trans_id, t.user_info.cust_id as cust_id,
> . . . . . . . > t.user_info.device as device, t.user_info.state as state,
> . . . . . . . > t.marketing_info.camp_id as camp_id,
> t.marketing_info.searchwords,
> . . . . . . . > t.trans_info.prod_id as prod_id, t.trans_info.purch_flag as
> purch_flag from
> . . . . . . . > click.`json_large`.`/nested.json` t;
> Query failed: Screen received stop request sent. Line 156, Column 32: A
> method named "copyAsValue" is not declared in any enclosing class nor any
> supertype, nor through a static import [d1668742-dd28-477d-a941-d03f0226d4e4]
> Node details: ip-172-16-1-111:31011/31012
> org.apache.drill.exec.exception.SchemaChangeException: Failure while
> attempting to load generated class
> at
> org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.setupNewSchema(ProjectRecordBatch.java:371)
> at
> org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:66)
> at
> org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.innerNext(ProjectRecordBatch.java:120)
> at
> org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:95)
> at
> org.apache.drill.exec.physical.impl.validate.IteratorValidatorBatchIterator.next(IteratorValidatorBatchIterator.java:116)
> at
> org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:59)
> at
> org.apache.drill.exec.physical.impl.ScreenCreator$ScreenRoot.innerNext(ScreenCreator.java:98)
> at
> org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:49)
> at
> org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:116)
> at
> org.apache.drill.exec.work.WorkManager$RunnableWrapper.run(WorkManager.java:250)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.drill.exec.exception.ClassTransformationException:
> Failure generating transformation classes for value:
>
> package org.apache.drill.exec.test.generated;
> import org.apache.drill.exec.exception.SchemaChangeException;
> import org.apache.drill.exec.expr.holders.BitHolder;
> import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
> import org.apache.drill.exec.expr.holders.RepeatedVarCharHolder;
> import org.apache.drill.exec.ops.FragmentContext;
> import org.apache.drill.exec.record.RecordBatch;
> import org.apache.drill.exec.vector.NullableVarCharVector;
> import org.apache.drill.exec.vector.RepeatedVarCharVector;
> import org.apache.drill.exec.vector.complex.impl.RepeatedVarCharWriterImpl;
> import org.apache.drill.exec.vector.complex.writer.VarCharWriter;
> public class ProjectorGen302 {
> NullableVarCharVector vv0;
> NullableVarCharVector vv4;
> NullableVarCharVector vv8;
> NullableVarCharVector vv12;
> RepeatedVarCharVector vv16;
> RepeatedVarCharVector vv20;
> VarCharWriter writer23;
> NullableVarCharVector vv25;
> NullableVarCharVector vv29;
> public void doSetup(FragmentContext context, RecordBatch incoming,
> RecordBatch outgoing)
> throws SchemaChangeException
> {
> {
> int[] fieldIds1 = new int[ 2 ] ;
> fieldIds1 [ 0 ] = 3;
> fieldIds1 [ 1 ] = 1;
> Object tmp2 =
> (incoming).getValueAccessorById(NullableVarCharVector.class,
> fieldIds1).getValueVector();
> if (tmp2 == null) {
> throw new SchemaChangeException("Failure while loading vector
> vv0 with id: org.apache.drill.exec.record.TypedFieldId@7dc6bfa3.");
> }
> vv0 = ((NullableVarCharVector) tmp2);
> int[] fieldIds5 = new int[ 1 ] ;
> fieldIds5 [ 0 ] = 2;
> Object tmp6 =
> (outgoing).getValueAccessorById(NullableVarCharVector.class,
> fieldIds5).getValueVector();
> if (tmp6 == null) {
> throw new SchemaChangeException("Failure while loading vector
> vv4 with id: org.apache.drill.exec.record.TypedFieldId@adcdf4e1.");
> }
> vv4 = ((NullableVarCharVector) tmp6);
> }
> {
> int[] fieldIds9 = new int[ 2 ] ;
> fieldIds9 [ 0 ] = 3;
> fieldIds9 [ 1 ] = 2;
> Object tmp10 =
> (incoming).getValueAccessorById(NullableVarCharVector.class,
> fieldIds9).getValueVector();
> if (tmp10 == null) {
> throw new SchemaChangeException("Failure while loading vector
> vv8 with id: org.apache.drill.exec.record.TypedFieldId@7f7b9842.");
> }
> vv8 = ((NullableVarCharVector) tmp10);
> int[] fieldIds13 = new int[ 1 ] ;
> fieldIds13 [ 0 ] = 3;
> Object tmp14 =
> (outgoing).getValueAccessorById(NullableVarCharVector.class,
> fieldIds13).getValueVector();
> if (tmp14 == null) {
> throw new SchemaChangeException("Failure while loading vector
> vv12 with id: org.apache.drill.exec.record.TypedFieldId@af82cd80.");
> }
> vv12 = ((NullableVarCharVector) tmp14);
> }
> {
> int[] fieldIds17 = new int[ 2 ] ;
> fieldIds17 [ 0 ] = 4;
> fieldIds17 [ 1 ] = 1;
> Object tmp18 =
> (incoming).getValueAccessorById(RepeatedVarCharVector.class,
> fieldIds17).getValueVector();
> if (tmp18 == null) {
> throw new SchemaChangeException("Failure while loading vector
> vv16 with id: org.apache.drill.exec.record.TypedFieldId@b5f8abde.");
> }
> vv16 = ((RepeatedVarCharVector) tmp18);
> int[] fieldIds21 = new int[ 1 ] ;
> fieldIds21 [ 0 ] = 5;
> Object tmp22 =
> (outgoing).getValueAccessorById(RepeatedVarCharVector.class,
> fieldIds21).getValueVector();
> if (tmp22 == null) {
> throw new SchemaChangeException("Failure while loading vector
> vv20 with id: org.apache.drill.exec.record.TypedFieldId@b6382fb8.");
> }
> vv20 = ((RepeatedVarCharVector) tmp22);
> writer23 = new RepeatedVarCharWriterImpl(vv20, null);
> }
> {
> int[] fieldIds26 = new int[ 2 ] ;
> fieldIds26 [ 0 ] = 5;
> fieldIds26 [ 1 ] = 1;
> Object tmp27 =
> (incoming).getValueAccessorById(NullableVarCharVector.class,
> fieldIds26).getValueVector();
> if (tmp27 == null) {
> throw new SchemaChangeException("Failure while loading vector
> vv25 with id: org.apache.drill.exec.record.TypedFieldId@e7933625.");
> }
> vv25 = ((NullableVarCharVector) tmp27);
> int[] fieldIds30 = new int[ 1 ] ;
> fieldIds30 [ 0 ] = 7;
> Object tmp31 =
> (outgoing).getValueAccessorById(NullableVarCharVector.class,
> fieldIds30).getValueVector();
> if (tmp31 == null) {
> throw new SchemaChangeException("Failure while loading vector
> vv29 with id: org.apache.drill.exec.record.TypedFieldId@b6562ffc.");
> }
> vv29 = ((NullableVarCharVector) tmp31);
> }
> }
> public boolean doEval(int inIndex, int outIndex)
> throws SchemaChangeException
> {
> {
> NullableVarCharHolder out3 = new NullableVarCharHolder();
> out3 .isSet = vv0 .getAccessor().isSet((inIndex));
> if (out3 .isSet == 1) {
> {
> vv0 .getAccessor().get((inIndex), out3);
> }
> }
> BitHolder out7 = new BitHolder();
> out7 .value = 1;
> if (!(out3 .isSet == 0)) {
> if (!vv4 .getMutator().setSafe((outIndex), out3)) {
> out7 .value = 0;
> }
> } else {
> if (!vv4 .getMutator().isSafe((outIndex))) {
> out7 .value = 0;
> }
> }
> if (out7 .value == 0) {
> return false;
> }
> }
> {
> NullableVarCharHolder out11 = new NullableVarCharHolder();
> out11 .isSet = vv8 .getAccessor().isSet((inIndex));
> if (out11 .isSet == 1) {
> {
> vv8 .getAccessor().get((inIndex), out11);
> }
> }
> BitHolder out15 = new BitHolder();
> out15 .value = 1;
> if (!(out11 .isSet == 0)) {
> if (!vv12 .getMutator().setSafe((outIndex), out11)) {
> out15 .value = 0;
> }
> } else {
> if (!vv12 .getMutator().isSafe((outIndex))) {
> out15 .value = 0;
> }
> }
> if (out15 .value == 0) {
> Error: exception while executing query: Failure while trying to get
> next result batch. (state=,code=0) return false;
> }
> }
> {
> RepeatedVarCharHolder out19 = new RepeatedVarCharHolder();
> {
> vv16 .getAccessor().get((inIndex), out19);
> }
> writer23 .setPosition((outIndex));
> out19 .copyAsValue(writer23);
> BitHolder out24 = new BitHolder();
> if (writer23 .ok()) {
> out24 .value = 1;
> } else {
> out24 .value = 0;
> }
> if (out24 .value == 0) {
> return false;
> }
> }
> {
> NullableVarCharHolder out28 = new NullableVarCharHolder();
> out28 .isSet = vv25 .getAccessor().isSet((inIndex));
> if (out28 .isSet == 1) {
> {
> vv25 .getAccessor().get((inIndex), out28);
> }
> }
> BitHolder out32 = new BitHolder();
> out32 .value = 1;
> if (!(out28 .isSet == 0)) {
> if (!vv29 .getMutator().setSafe((outIndex), out28)) {
> out32 .value = 0;
> }
> } else {
> if (!vv29 .getMutator().isSafe((outIndex))) {
> out32 .value = 0;
> }
> }
> if (out32 .value == 0) {
> return false;
> }
> }
> {
> return true;
> }
> }
> }
> at
> org.apache.drill.exec.compile.ClassTransformer.getImplementationClass(ClassTransformer.java:295)
> at
> org.apache.drill.exec.ops.FragmentContext.getImplementationClass(FragmentContext.java:187)
> at
> org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.setupNewSchema(ProjectRecordBatch.java:368)
> ... 12 more
> Caused by: org.codehaus.commons.compiler.CompileException: Line 156, Column
> 32: A method named "copyAsValue" is not declared in any enclosing class nor
> any supertype, nor through a static import
> at
> org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:10056)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:7298)
> at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:3860)
> at org.codehaus.janino.UnitCompiler.access$6900(UnitCompiler.java:182)
> at
> org.codehaus.janino.UnitCompiler$10.visitMethodInvocation(UnitCompiler.java:3261)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:3978)
> at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3288)
> at
> org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4354)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2662)
> at org.codehaus.janino.UnitCompiler.access$4400(UnitCompiler.java:182)
> at
> org.codehaus.janino.UnitCompiler$7.visitMethodInvocation(UnitCompiler.java:2627)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:3978)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2654)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1634)
> at org.codehaus.janino.UnitCompiler.access$1100(UnitCompiler.java:182)
> at
> org.codehaus.janino.UnitCompiler$4.visitExpressionStatement(UnitCompiler.java:940)
> at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2085)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:962)
> at
> org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1004)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:989)
> at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:182)
> at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:939)
> at org.codehaus.janino.Java$Block.accept(Java.java:2005)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:962)
> at
> org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1004)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2284)
> at
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:826)
> at
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:798)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:503)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:389)
> at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:182)
> at
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:343)
> at
> org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1136)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:350)
> at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:318)
> at
> org.apache.drill.exec.compile.JaninoClassCompiler.getByteCode(JaninoClassCompiler.java:49)
> at
> org.apache.drill.exec.compile.AbstractClassCompiler.getClassByteCode(AbstractClassCompiler.java:39)
> at
> org.apache.drill.exec.compile.QueryClassLoader$ClassCompilerSelector.getClassByteCode(QueryClassLoader.java:140)
> at
> org.apache.drill.exec.compile.QueryClassLoader$ClassCompilerSelector.access$000(QueryClassLoader.java:109)
> at
> org.apache.drill.exec.compile.QueryClassLoader.getClassByteCode(QueryClassLoader.java:102)
> at
> org.apache.drill.exec.compile.ClassTransformer.getImplementationClass(ClassTransformer.java:254)
> ... 14 more
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
