Can you file a Drill JIRA? From the stack trace and the relevant code, seems there is a bug in HiveMetadataProvider.
sizeInBytes = Long.valueOf(numRowsProp) ==> sizeInBytes = Long.valueOf(sizeInBytesProp); 1. https://github.com/apache/drill/blob/master/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java#L209-L211 On Tue, Feb 7, 2017 at 10:40 AM, rahul challapalli <[email protected]> wrote: > Your query is the longest query I have heard of :) > > In any case, lets try the below steps : > > 1. Can you first try your query directly on hive? If hive reports an error > during its metadata operations, then you can expect drill to fail as well > during planning. > 2. Increase the heap memory : Since the query is failing at planning stage, > lets give the planner more memory and see if we can make any progress > > - Rahul > > > On Tue, Feb 7, 2017 at 7:55 AM, <[email protected]> wrote: > >> Hi all, >> >> I hope this is the correct place to ask for help. >> >> We have some hive tables stored as textfile that we try to query from, and >> perform some joins. >> Because we have programmed a SQL Query generator and based on end user’s >> selection, it came up with a SQL query that is about 1 million characters >> long and has about 700 selects/nested queries. >> After submitting the query thru Drill’s web interface, the query is >> accepted but stuck in Starting state. Further checks from log file, we got >> the following. >> We are running a 2 node drill cluster, m4.2xlarge 8 vCPU and 32GB of ram. >> >> Thus wanted to check if anyone has ran a long query like ours before or we >> should be looking at another area on the issue. >> >> Thank you in advance! >> >> 2017-02-07 15:25:45,995 [27661a58-2d7e-650c-0000-b1c82b38bb9a:foreman] >> ERROR o.a.d.e.s.hive.HiveMetadataProvider - Failed to parse Hive stats in >> metastore. >> java.lang.NumberFormatException: null >> at java.lang.Long.parseLong(Long.java:552) ~[na:1.8.0_111] >> at java.lang.Long.valueOf(Long.java:803) ~[na:1.8.0_111] >> at org.apache.drill.exec.store.hive.HiveMetadataProvider. >> getStatsFromProps(HiveMetadataProvider.java:212) >> [drill-storage-hive-core-1.9.0.jar:1.9.0] >> at org.apache.drill.exec.store.hive.HiveMetadataProvider. >> getStats(HiveMetadataProvider.java:90) [drill-storage-hive-core-1.9. >> 0.jar:1.9.0] >> at >> org.apache.drill.exec.store.hive.HiveScan.getScanStats(HiveScan.java:224) >> [drill-storage-hive-core-1.9.0.jar:1.9.0] >> at org.apache.drill.exec.physical.base.AbstractGroupScan. >> getScanStats(AbstractGroupScan.java:79) [drill-java-exec-1.9.0.jar:1.9.0] >> at org.apache.drill.exec.planner.logical.DrillScanRel. >> computeSelfCost(DrillScanRel.java:159) [drill-java-exec-1.9.0.jar:1.9.0] >> at org.apache.calcite.rel.metadata.RelMdPercentageOriginalRows. >> getNonCumulativeCost(RelMdPercentageOriginalRows.java:165) >> [calcite-core-1.4.0-drill-r19.jar:1.4.0-drill-r19] >> at sun.reflect.GeneratedMethodAccessor12.invoke(Unknown Source) >> ~[na:na] >> at sun.reflect.DelegatingMethodAccessorImpl.invoke( >> DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_111] >> at java.lang.reflect.Method.invoke(Method.java:498) >> ~[na:1.8.0_111] >> at org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider$ >> 1$1.invoke(ReflectiveRelMetadataProvider.java:182) >> [calcite-core-1.4.0-drill-r19.jar:1.4.0-drill-r19] >> at com.sun.proxy.$Proxy71.getNonCumulativeCost(Unknown Source) >> [na:na] >> at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source) >> ~[na:na] >> at sun.reflect.DelegatingMethodAccessorImpl.invoke( >> DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_111] >> at java.lang.reflect.Method.invoke(Method.java:498) >> ~[na:1.8.0_111] >> at org.apache.calcite.rel.metadata.ChainedRelMetadataProvider$ >> ChainedInvocationHandler.invoke(ChainedRelMetadataProvider.java:109) >> [calcite-core-1.4.0-drill-r19.jar:1.4.0-drill-r19] >> at com.sun.proxy.$Proxy71.getNonCumulativeCost(Unknown Source) >> [na:na] >> at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source) >> ~[na:na] >> at sun.reflect.DelegatingMethodAccessorImpl.invoke( >> DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_111] >> at java.lang.reflect.Method.invoke(Method.java:498) >> ~[na:1.8.0_111] >> at org.apache.calcite.rel.metadata.CachingRelMetadataProvider$ >> CachingInvocationHandler.invoke(CachingRelMetadataProvider.java:132) >> [calcite-core-1.4.0-drill-r19.jar:1.4.0-drill-r19] >> at com.sun.proxy.$Proxy71.getNonCumulativeCost(Unknown Source) >> [na:na] >> at org.apache.calcite.rel.metadata.RelMetadataQuery. >> getNonCumulativeCost(RelMetadataQuery.java:115) >> [calcite-core-1.4.0-drill-r19.jar:1.4.0-drill-r19] >> at org.apache.calcite.plan.volcano.VolcanoPlanner. >> getCost(VolcanoPlanner.java:1112) [calcite-core-1.4.0-drill-r19. >> jar:1.4.0-drill-r19] >> at org.apache.calcite.plan.volcano.RelSubset. >> propagateCostImprovements0(RelSubset.java:363) >> [calcite-core-1.4.0-drill-r19.jar:1.4.0-drill-r19] >> at org.apache.calcite.plan.volcano.RelSubset. >> propagateCostImprovements(RelSubset.java:344) >> [calcite-core-1.4.0-drill-r19.jar:1.4.0-drill-r19] >> at org.apache.calcite.plan.volcano.VolcanoPlanner. >> addRelToSet(VolcanoPlanner.java:1827) [calcite-core-1.4.0-drill-r19. >> jar:1.4.0-drill-r19] >> at org.apache.calcite.plan.volcano.VolcanoPlanner. >> registerImpl(VolcanoPlanner.java:1760) [calcite-core-1.4.0-drill-r19. >> jar:1.4.0-drill-r19] >> at org.apache.calcite.plan.volcano.VolcanoPlanner. >> register(VolcanoPlanner.java:1017) [calcite-core-1.4.0-drill-r19. >> jar:1.4.0-drill-r19] >> at org.apache.calcite.plan.volcano.VolcanoPlanner. >> ensureRegistered(VolcanoPlanner.java:1037) [calcite-core-1.4.0-drill-r19. >> jar:1.4.0-drill-r19] >> at org.apache.calcite.plan.volcano.VolcanoPlanner. >> ensureRegistered(VolcanoPlanner.java:1940) [calcite-core-1.4.0-drill-r19. >> jar:1.4.0-drill-r19] >> at org.apache.calcite.plan.volcano.VolcanoRuleCall. >> transformTo(VolcanoRuleCall.java:138) [calcite-core-1.4.0-drill-r19. >> jar:1.4.0-drill-r19] >> at >> org.apache.calcite.plan.RelOptRuleCall.transformTo(RelOptRuleCall.java:213) >> [calcite-core-1.4.0-drill-r19.jar:1.4.0-drill-r19] >> at org.apache.drill.exec.planner.logical.DrillPushProjIntoScan. >> onMatch(DrillPushProjIntoScan.java:90) [drill-java-exec-1.9.0.jar:1.9.0] >> at org.apache.calcite.plan.volcano.VolcanoRuleCall. >> onMatch(VolcanoRuleCall.java:228) [calcite-core-1.4.0-drill-r19. >> jar:1.4.0-drill-r19] >> at org.apache.calcite.plan.volcano.VolcanoPlanner. >> findBestExp(VolcanoPlanner.java:808) [calcite-core-1.4.0-drill-r19. >> jar:1.4.0-drill-r19] >> at >> org.apache.calcite.tools.Programs$RuleSetProgram.run(Programs.java:303) >> [calcite-core-1.4.0-drill-r19.jar:1.4.0-drill-r19] >> at org.apache.drill.exec.planner.sql.handlers. >> DefaultSqlHandler.transform(DefaultSqlHandler.java:404) >> [drill-java-exec-1.9.0.jar:1.9.0] >> at org.apache.drill.exec.planner.sql.handlers. >> DefaultSqlHandler.transform(DefaultSqlHandler.java:343) >> [drill-java-exec-1.9.0.jar:1.9.0] >> at org.apache.drill.exec.planner.sql.handlers.DefaultSqlHandler. >> convertToDrel(DefaultSqlHandler.java:240) [drill-java-exec-1.9.0.jar:1. >> 9.0] >> at org.apache.drill.exec.planner.sql.handlers.DefaultSqlHandler. >> convertToDrel(DefaultSqlHandler.java:290) [drill-java-exec-1.9.0.jar:1. >> 9.0] >> at org.apache.drill.exec.planner.sql.handlers. >> DefaultSqlHandler.getPlan(DefaultSqlHandler.java:168) >> [drill-java-exec-1.9.0.jar:1.9.0] >> at org.apache.drill.exec.planner.sql.DrillSqlWorker. >> getPhysicalPlan(DrillSqlWorker.java:123) [drill-java-exec-1.9.0.jar:1.9.0] >> at >> org.apache.drill.exec.planner.sql.DrillSqlWorker.getPlan(DrillSqlWorker.java:97) >> [drill-java-exec-1.9.0.jar:1.9.0] >> at >> org.apache.drill.exec.work.foreman.Foreman.runSQL(Foreman.java:1008) >> [drill-java-exec-1.9.0.jar:1.9.0] >> at org.apache.drill.exec.work.foreman.Foreman.run(Foreman.java:264) >> [drill-java-exec-1.9.0.jar:1.9.0] >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) >> [na:1.8.0_111] >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> [na:1.8.0_111] >> at java.lang.Thread.run(Thread.java:745) [na:1.8.0_111] >> >> Kind Regards, >> >> Chua Tianxiang >>
