[jira] [Commented] (HIVE-17272) when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE
[ https://issues.apache.org/jira/browse/HIVE-17272?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16279988#comment-16279988 ] Wang Haihua commented on HIVE-17272: +1 we found this NPE happens for vectorization too. > when hive.vectorized.execution.enabled is true, query on empty partitioned > table fails with NPE > --- > > Key: HIVE-17272 > URL: https://issues.apache.org/jira/browse/HIVE-17272 > Project: Hive > Issue Type: Bug > Components: Query Planning >Affects Versions: 2.1.1 >Reporter: Aihua Xu >Assignee: Aihua Xu > Fix For: 3.0.0, 2.4.0 > > Attachments: HIVE-17272.2.patch > > > {noformat} > set hive.vectorized.execution.enabled=true; > CREATE TABLE `tab`(`x` int) PARTITIONED BY ( `y` int) stored as parquet; > select * from tab t1 join tab t2 where t1.x=t2.x; > {noformat} > The query fails with the following exception. > {noformat} > Caused by: java.lang.NullPointerException > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.createAndInitPartitionContext(VectorMapOperator.java:386) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.internalSetChildren(VectorMapOperator.java:559) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.setChildren(VectorMapOperator.java:474) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.mr.ExecMapper.configure(ExecMapper.java:106) > ~[hive-exec-2.3.0.jar:2.3.0] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:413) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:268) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[?:1.8.0_101] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > ~[?:1.8.0_101] > at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_101] > {noformat} -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Commented] (HIVE-17272) when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE
[ https://issues.apache.org/jira/browse/HIVE-17272?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16243176#comment-16243176 ] Vihang Karajgaonkar commented on HIVE-17272: Patch merged to branch-2 as well. The backport was clean and I was able to run the test from the patch without any changes. > when hive.vectorized.execution.enabled is true, query on empty partitioned > table fails with NPE > --- > > Key: HIVE-17272 > URL: https://issues.apache.org/jira/browse/HIVE-17272 > Project: Hive > Issue Type: Bug > Components: Query Planning >Affects Versions: 2.1.1 >Reporter: Aihua Xu >Assignee: Aihua Xu > Fix For: 3.0.0, 2.4.0 > > Attachments: HIVE-17272.2.patch > > > {noformat} > set hive.vectorized.execution.enabled=true; > CREATE TABLE `tab`(`x` int) PARTITIONED BY ( `y` int) stored as parquet; > select * from tab t1 join tab t2 where t1.x=t2.x; > {noformat} > The query fails with the following exception. > {noformat} > Caused by: java.lang.NullPointerException > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.createAndInitPartitionContext(VectorMapOperator.java:386) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.internalSetChildren(VectorMapOperator.java:559) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.setChildren(VectorMapOperator.java:474) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.mr.ExecMapper.configure(ExecMapper.java:106) > ~[hive-exec-2.3.0.jar:2.3.0] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:413) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:268) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[?:1.8.0_101] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > ~[?:1.8.0_101] > at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_101] > {noformat} -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Commented] (HIVE-17272) when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE
[ https://issues.apache.org/jira/browse/HIVE-17272?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16128999#comment-16128999 ] Aihua Xu commented on HIVE-17272: - [~ngangam] With my fix above, we shouldn't run into NPE in the place you mentioned above since it returns before calling init() from createAndInitPartitionContext(). > when hive.vectorized.execution.enabled is true, query on empty partitioned > table fails with NPE > --- > > Key: HIVE-17272 > URL: https://issues.apache.org/jira/browse/HIVE-17272 > Project: Hive > Issue Type: Bug > Components: Query Planning >Affects Versions: 2.1.1 >Reporter: Aihua Xu >Assignee: Aihua Xu > Attachments: HIVE-17272.2.patch > > > {noformat} > set hive.vectorized.execution.enabled=true; > CREATE TABLE `tab`(`x` int) PARTITIONED BY ( `y` int) stored as parquet; > select * from tab t1 join tab t2 where t1.x=t2.x; > {noformat} > The query fails with the following exception. > {noformat} > Caused by: java.lang.NullPointerException > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.createAndInitPartitionContext(VectorMapOperator.java:386) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.internalSetChildren(VectorMapOperator.java:559) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.setChildren(VectorMapOperator.java:474) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.mr.ExecMapper.configure(ExecMapper.java:106) > ~[hive-exec-2.3.0.jar:2.3.0] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:413) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:268) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[?:1.8.0_101] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > ~[?:1.8.0_101] > at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_101] > {noformat} -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Commented] (HIVE-17272) when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE
[ https://issues.apache.org/jira/browse/HIVE-17272?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16128971#comment-16128971 ] Aihua Xu commented on HIVE-17272: - [~ngangam] I don't see a case which will fail in that function. Do you have one? > when hive.vectorized.execution.enabled is true, query on empty partitioned > table fails with NPE > --- > > Key: HIVE-17272 > URL: https://issues.apache.org/jira/browse/HIVE-17272 > Project: Hive > Issue Type: Bug > Components: Query Planning >Affects Versions: 2.1.1 >Reporter: Aihua Xu >Assignee: Aihua Xu > Attachments: HIVE-17272.2.patch > > > {noformat} > set hive.vectorized.execution.enabled=true; > CREATE TABLE `tab`(`x` int) PARTITIONED BY ( `y` int) stored as parquet; > select * from tab t1 join tab t2 where t1.x=t2.x; > {noformat} > The query fails with the following exception. > {noformat} > Caused by: java.lang.NullPointerException > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.createAndInitPartitionContext(VectorMapOperator.java:386) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.internalSetChildren(VectorMapOperator.java:559) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.setChildren(VectorMapOperator.java:474) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.mr.ExecMapper.configure(ExecMapper.java:106) > ~[hive-exec-2.3.0.jar:2.3.0] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:413) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:268) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[?:1.8.0_101] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > ~[?:1.8.0_101] > at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_101] > {noformat} -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Commented] (HIVE-17272) when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE
[ https://issues.apache.org/jira/browse/HIVE-17272?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16128774#comment-16128774 ] Naveen Gangam commented on HIVE-17272: -- [~aihuaxu] Appears NPE could occur in both the init() methods as well. Have you investigated these as well? {code} public void init(Configuration hconf) throws Exception { VectorPartitionDesc vectorPartDesc = partDesc.getVectorPartitionDesc(); ... TypeInfo[] dataTypeInfos = vectorPartDesc.getDataTypeInfos(); {code} Thanks > when hive.vectorized.execution.enabled is true, query on empty partitioned > table fails with NPE > --- > > Key: HIVE-17272 > URL: https://issues.apache.org/jira/browse/HIVE-17272 > Project: Hive > Issue Type: Bug > Components: Query Planning >Affects Versions: 2.1.1 >Reporter: Aihua Xu >Assignee: Aihua Xu > Attachments: HIVE-17272.2.patch > > > {noformat} > set hive.vectorized.execution.enabled=true; > CREATE TABLE `tab`(`x` int) PARTITIONED BY ( `y` int) stored as parquet; > select * from tab t1 join tab t2 where t1.x=t2.x; > {noformat} > The query fails with the following exception. > {noformat} > Caused by: java.lang.NullPointerException > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.createAndInitPartitionContext(VectorMapOperator.java:386) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.internalSetChildren(VectorMapOperator.java:559) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.setChildren(VectorMapOperator.java:474) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.mr.ExecMapper.configure(ExecMapper.java:106) > ~[hive-exec-2.3.0.jar:2.3.0] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:413) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:268) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[?:1.8.0_101] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > ~[?:1.8.0_101] > at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_101] > {noformat} -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Commented] (HIVE-17272) when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE
[ https://issues.apache.org/jira/browse/HIVE-17272?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16128126#comment-16128126 ] Vihang Karajgaonkar commented on HIVE-17272: +1 LGTM. I think the other way to fix this would have been in {{Vectorizer#validateInputFormatAndSchemaEvolution}} and return false if {{pathToPartitionInfo}} is empty. > when hive.vectorized.execution.enabled is true, query on empty partitioned > table fails with NPE > --- > > Key: HIVE-17272 > URL: https://issues.apache.org/jira/browse/HIVE-17272 > Project: Hive > Issue Type: Bug > Components: Query Planning >Affects Versions: 2.1.1 >Reporter: Aihua Xu >Assignee: Aihua Xu > Attachments: HIVE-17272.2.patch > > > {noformat} > set hive.vectorized.execution.enabled=true; > CREATE TABLE `tab`(`x` int) PARTITIONED BY ( `y` int) stored as parquet; > select * from tab t1 join tab t2 where t1.x=t2.x; > {noformat} > The query fails with the following exception. > {noformat} > Caused by: java.lang.NullPointerException > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.createAndInitPartitionContext(VectorMapOperator.java:386) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.internalSetChildren(VectorMapOperator.java:559) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.setChildren(VectorMapOperator.java:474) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.mr.ExecMapper.configure(ExecMapper.java:106) > ~[hive-exec-2.3.0.jar:2.3.0] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:413) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:268) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[?:1.8.0_101] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > ~[?:1.8.0_101] > at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_101] > {noformat} -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Commented] (HIVE-17272) when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE
[ https://issues.apache.org/jira/browse/HIVE-17272?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16127515#comment-16127515 ] Aihua Xu commented on HIVE-17272: - patch-2: handle the case of vectorPartDesc is null to avoid NPE exception. Such case can happen for the empty table while internally hive is generating an empty file for it. > when hive.vectorized.execution.enabled is true, query on empty partitioned > table fails with NPE > --- > > Key: HIVE-17272 > URL: https://issues.apache.org/jira/browse/HIVE-17272 > Project: Hive > Issue Type: Bug > Components: Query Planning >Affects Versions: 2.1.1 >Reporter: Aihua Xu >Assignee: Aihua Xu > Attachments: HIVE-17272.2.patch > > > {noformat} > set hive.vectorized.execution.enabled=true; > CREATE TABLE `tab`(`x` int) PARTITIONED BY ( `y` int) stored as parquet; > select * from tab t1 join tab t2 where t1.x=t2.x; > {noformat} > The query fails with the following exception. > {noformat} > Caused by: java.lang.NullPointerException > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.createAndInitPartitionContext(VectorMapOperator.java:386) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.internalSetChildren(VectorMapOperator.java:559) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.setChildren(VectorMapOperator.java:474) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.mr.ExecMapper.configure(ExecMapper.java:106) > ~[hive-exec-2.3.0.jar:2.3.0] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:413) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:268) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[?:1.8.0_101] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > ~[?:1.8.0_101] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > ~[?:1.8.0_101] > at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_101] > {noformat} -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Commented] (HIVE-17272) when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE
[ https://issues.apache.org/jira/browse/HIVE-17272?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16126421#comment-16126421 ] Hive QA commented on HIVE-17272: Here are the results of testing the latest attachment: https://issues.apache.org/jira/secure/attachment/12881815/HIVE-17272.2.patch {color:green}SUCCESS:{color} +1 due to 1 test(s) being added or modified. {color:red}ERROR:{color} -1 due to 10 failed/errored test(s), 11004 tests executed *Failed tests:* {noformat} org.apache.hadoop.hive.cli.TestBlobstoreCliDriver.testCliDriver[insert_overwrite_dynamic_partitions_merge_move] (batchId=243) org.apache.hadoop.hive.cli.TestBlobstoreCliDriver.testCliDriver[insert_overwrite_dynamic_partitions_merge_only] (batchId=243) org.apache.hadoop.hive.cli.TestBlobstoreCliDriver.testCliDriver[insert_overwrite_dynamic_partitions_move_only] (batchId=243) org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[spark_dynamic_partition_pruning] (batchId=169) org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[spark_dynamic_partition_pruning_mapjoin_only] (batchId=170) org.apache.hadoop.hive.cli.TestMiniSparkOnYarnCliDriver.testCliDriver[spark_vectorized_dynamic_partition_pruning] (batchId=169) org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[explainanalyze_3] (batchId=99) org.apache.hive.hcatalog.api.TestHCatClient.testPartitionRegistrationWithCustomSchema (batchId=180) org.apache.hive.hcatalog.api.TestHCatClient.testPartitionSpecRegistrationWithCustomSchema (batchId=180) org.apache.hive.hcatalog.api.TestHCatClient.testTableSchemaPropagation (batchId=180) {noformat} Test results: https://builds.apache.org/job/PreCommit-HIVE-Build/6388/testReport Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/6388/console Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-6388/ Messages: {noformat} Executing org.apache.hive.ptest.execution.TestCheckPhase Executing org.apache.hive.ptest.execution.PrepPhase Executing org.apache.hive.ptest.execution.ExecutionPhase Executing org.apache.hive.ptest.execution.ReportingPhase Tests exited with: TestsFailedException: 10 tests failed {noformat} This message is automatically generated. ATTACHMENT ID: 12881815 - PreCommit-HIVE-Build > when hive.vectorized.execution.enabled is true, query on empty partitioned > table fails with NPE > --- > > Key: HIVE-17272 > URL: https://issues.apache.org/jira/browse/HIVE-17272 > Project: Hive > Issue Type: Bug > Components: Query Planning >Affects Versions: 2.1.1 >Reporter: Aihua Xu >Assignee: Aihua Xu > Attachments: HIVE-17272.1.patch, HIVE-17272.2.patch > > > {noformat} > set hive.vectorized.execution.enabled=true; > CREATE TABLE `tab`(`x` int) PARTITIONED BY ( `y` int) stored as parquet; > select * from tab t1 join tab t2 where t1.x=t2.x; > {noformat} > The query fails with the following exception. > {noformat} > Caused by: java.lang.NullPointerException > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.createAndInitPartitionContext(VectorMapOperator.java:386) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.internalSetChildren(VectorMapOperator.java:559) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.setChildren(VectorMapOperator.java:474) > ~[hive-exec-2.3.0.jar:2.3.0] > at > org.apache.hadoop.hive.ql.exec.mr.ExecMapper.configure(ExecMapper.java:106) > ~[hive-exec-2.3.0.jar:2.3.0] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] > at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_101] > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:75) > ~[hadoop-common-2.6.0.jar:?] > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > ~[hadoop-common-2.6.0.jar:?] > at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) > ~[hadoop-core-2.6.0-mr1-cdh5.4.2.jar:?] > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > ~[?:1.8.0_101] > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > ~[?:1.8.0_101] > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > ~[?:1.8.0_101] >
[jira] [Commented] (HIVE-17272) when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE
[ https://issues.apache.org/jira/browse/HIVE-17272?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16124816#comment-16124816 ] Hive QA commented on HIVE-17272: Here are the results of testing the latest attachment: https://issues.apache.org/jira/secure/attachment/12881639/HIVE-17272.1.patch {color:red}ERROR:{color} -1 due to no test(s) being added or modified. {color:red}ERROR:{color} -1 due to 381 failed/errored test(s), 10966 tests executed *Failed tests:* {noformat} TestTxnCommands2WithSplitUpdateAndVectorization - did not produce a TEST-*.xml file (likely timed out) (batchId=280) org.apache.hadoop.hive.cli.TestBlobstoreCliDriver.testCliDriver[insert_overwrite_dynamic_partitions_merge_move] (batchId=243) org.apache.hadoop.hive.cli.TestBlobstoreCliDriver.testCliDriver[insert_overwrite_dynamic_partitions_merge_only] (batchId=243) org.apache.hadoop.hive.cli.TestBlobstoreCliDriver.testCliDriver[insert_overwrite_dynamic_partitions_move_only] (batchId=243) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[acid_vectorization] (batchId=62) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[acid_vectorization_partition] (batchId=69) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[acid_vectorization_project] (batchId=19) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[constprog_when_case] (batchId=56) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[foldts] (batchId=55) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[llap_acid] (batchId=77) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[llap_reader] (batchId=7) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[llap_uncompressed] (batchId=56) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[mergejoin] (batchId=57) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[optimize_filter_literal] (batchId=51) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[orc_int_type_promotion] (batchId=40) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[orc_vectorization_ppd] (batchId=39) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parquet_no_row_serde] (batchId=67) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parquet_types_non_dictionary_encoding_vectorization] (batchId=81) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parquet_types_vectorization] (batchId=13) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[tez_join_hash] (batchId=50) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_acid3] (batchId=27) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_aggregate_9] (batchId=38) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_aggregate_without_gby] (batchId=51) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_between_columns] (batchId=66) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_binary_join_groupby] (batchId=78) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_bround] (batchId=32) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_bucket] (batchId=25) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_cast_constant] (batchId=8) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_char_2] (batchId=66) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_char_4] (batchId=85) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_char_cast] (batchId=63) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_char_mapjoin1] (batchId=31) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_char_simple] (batchId=45) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_coalesce] (batchId=10) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_coalesce_2] (batchId=68) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_coalesce_3] (batchId=55) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_complex_join] (batchId=42) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_const] (batchId=59) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_count] (batchId=13) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_custom_udf_configure] (batchId=63) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_data_types] (batchId=73) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_date_1] (batchId=20) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_10_0] (batchId=54) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_1] (batchId=76) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_3] (batchId=19) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_4] (batchId=28) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_5] (batchId=1) org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_6] (batchId=13)