[
https://issues.apache.org/jira/browse/HIVE-23960?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17168265#comment-17168265
]
Pravin Sinha commented on HIVE-23960:
-------------------------------------
The root cause for the imbalance if this exception:
{code:java}
2020-07-24 08:24:03,843 INFO DataNucleus.JDO: [pool-12-thread-190]: Exception
thrown
org.datanucleus.store.query.QueryCompilerSyntaxException: Method/Identifier
expected at character 95 in "tableName == t1 && dbName == t2 && catName == t3
&& engine == t4 && (partitionName == p4) && ()"
at
org.datanucleus.query.compiler.JDOQLParser.processPrimary(JDOQLParser.java:807)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processUnaryExpression(JDOQLParser.java:648)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processMultiplicativeExpression(JDOQLParser.java:574)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processAdditiveExpression(JDOQLParser.java:545)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processRelationalExpression(JDOQLParser.java:459)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processAndExpression(JDOQLParser.java:442)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processExclusiveOrExpression(JDOQLParser.java:428)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processInclusiveOrExpression(JDOQLParser.java:414)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processConditionalAndExpression(JDOQLParser.java:400)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processConditionalOrExpression(JDOQLParser.java:381)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processExpression(JDOQLParser.java:370)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processPrimary(JDOQLParser.java:782)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processUnaryExpression(JDOQLParser.java:648)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processMultiplicativeExpression(JDOQLParser.java:574)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processAdditiveExpression(JDOQLParser.java:545)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processRelationalExpression(JDOQLParser.java:459)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processAndExpression(JDOQLParser.java:442)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processExclusiveOrExpression(JDOQLParser.java:428)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processInclusiveOrExpression(JDOQLParser.java:414)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processConditionalAndExpression(JDOQLParser.java:404)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processConditionalOrExpression(JDOQLParser.java:381)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.processExpression(JDOQLParser.java:370)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLParser.parse(JDOQLParser.java:93)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JavaQueryCompiler.compileFilter(JavaQueryCompiler.java:525)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.query.compiler.JDOQLCompiler.compile(JDOQLCompiler.java:116)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.store.query.AbstractJDOQLQuery.compileGeneric(AbstractJDOQLQuery.java:370)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.store.query.AbstractJDOQLQuery.compileInternal(AbstractJDOQLQuery.java:415)
~[datanucleus-core-4.1.17.jar:?]
at
org.datanucleus.store.rdbms.query.JDOQLQuery.compileInternal(JDOQLQuery.java:239)
~[datanucleus-rdbms-4.1.19.jar:?]
at org.datanucleus.store.query.Query.executeQuery(Query.java:1816)
~[datanucleus-core-4.1.17.jar:?]
at org.datanucleus.store.query.Query.executeWithArray(Query.java:1744)
~[datanucleus-core-4.1.17.jar:?]
at org.datanucleus.api.jdo.JDOQuery.executeInternal(JDOQuery.java:368)
[datanucleus-api-jdo-4.2.4.jar:?]
at org.datanucleus.api.jdo.JDOQuery.executeWithArray(JDOQuery.java:267)
[datanucleus-api-jdo-4.2.4.jar:?]
at
org.apache.hadoop.hive.metastore.ObjectStore.getMPartitionColumnStatistics(ObjectStore.java:9749)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore.access$3200(ObjectStore.java:375)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore$21.getJdoResult(ObjectStore.java:9575)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore$21.getJdoResult(ObjectStore.java:9563)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore$GetHelper.run(ObjectStore.java:4122)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore.getPartitionColumnStatisticsInternal(ObjectStore.java:9606)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore.getPartitionColumnStatistics(ObjectStore.java:9517)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at jdk.internal.reflect.GeneratedMethodAccessor65.invoke(Unknown
Source) ~[?:?]
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:?]
at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at
org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:97)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at com.sun.proxy.$Proxy34.getPartitionColumnStatistics(Unknown Source)
[?:?]
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_partitions_by_names(HiveMetaStore.java:7021)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_partitions_by_names_req(HiveMetaStore.java:6977)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at jdk.internal.reflect.GeneratedMethodAccessor133.invoke(Unknown
Source) ~[?:?]
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:?]
at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:108)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at com.sun.proxy.$Proxy35.get_partitions_by_names_req(Unknown Source)
[?:?]
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$get_partitions_by_names_req.getResult(ThriftHiveMetastore.java:18270)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$get_partitions_by_names_req.getResult(ThriftHiveMetastore.java:18254)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge$Server$TUGIAssumingProcessor$1.run(HadoopThriftAuthBridge.java:643)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge$Server$TUGIAssumingProcessor$1.run(HadoopThriftAuthBridge.java:638)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at java.security.AccessController.doPrivileged(Native Method) [?:?]
at javax.security.auth.Subject.doAs(Subject.java:423) [?:?]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
[hadoop-common-3.1.1.7.2.1.0-258.jar:?]
at
org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge$Server$TUGIAssumingProcessor.process(HadoopThriftAuthBridge.java:638)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[?:?]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[?:?]
at java.lang.Thread.run(Thread.java:834) [?:?]
2020-07-24 08:24:03,845 ERROR org.apache.hadoop.hive.metastore.ObjectStore:
[pool-12-thread-190]: Error retrieving statistics via jdo
javax.jdo.JDOUserException: Method/Identifier expected at character 95 in
"tableName == t1 && dbName == t2 && catName == t3 && engine == t4 &&
(partitionName == p4) && ()"
at
org.datanucleus.api.jdo.NucleusJDOHelper.getJDOExceptionForNucleusException(NucleusJDOHelper.java:636)
~[datanucleus-api-jdo-4.2.4.jar:?]
at org.datanucleus.api.jdo.JDOQuery.executeInternal(JDOQuery.java:391)
~[datanucleus-api-jdo-4.2.4.jar:?]
at org.datanucleus.api.jdo.JDOQuery.executeWithArray(JDOQuery.java:267)
~[datanucleus-api-jdo-4.2.4.jar:?]
at
org.apache.hadoop.hive.metastore.ObjectStore.getMPartitionColumnStatistics(ObjectStore.java:9749)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore.access$3200(ObjectStore.java:375)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore$21.getJdoResult(ObjectStore.java:9575)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore$21.getJdoResult(ObjectStore.java:9563)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore$GetHelper.run(ObjectStore.java:4122)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore.getPartitionColumnStatisticsInternal(ObjectStore.java:9606)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.ObjectStore.getPartitionColumnStatistics(ObjectStore.java:9517)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at jdk.internal.reflect.GeneratedMethodAccessor65.invoke(Unknown
Source) ~[?:?]
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:?]
at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at
org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:97)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at com.sun.proxy.$Proxy34.getPartitionColumnStatistics(Unknown Source)
[?:?]
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_partitions_by_names(HiveMetaStore.java:7021)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_partitions_by_names_req(HiveMetaStore.java:6977)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at jdk.internal.reflect.GeneratedMethodAccessor133.invoke(Unknown
Source) ~[?:?]
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:?]
at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:108)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at com.sun.proxy.$Proxy35.get_partitions_by_names_req(Unknown Source)
[?:?]
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$get_partitions_by_names_req.getResult(ThriftHiveMetastore.java:18270)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$get_partitions_by_names_req.getResult(ThriftHiveMetastore.java:18254)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge$Server$TUGIAssumingProcessor$1.run(HadoopThriftAuthBridge.java:643)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge$Server$TUGIAssumingProcessor$1.run(HadoopThriftAuthBridge.java:638)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at java.security.AccessController.doPrivileged(Native Method) [?:?]
at javax.security.auth.Subject.doAs(Subject.java:423) [?:?]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
[hadoop-common-3.1.1.7.2.1.0-258.jar:?]
at
org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge$Server$TUGIAssumingProcessor.process(HadoopThriftAuthBridge.java:638)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)
[hive-exec-3.1.3000.7.2.1.0-258.jar:3.1.3000.7.2.1.0-258]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[?:?]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[?:?]
at java.lang.Thread.run(Thread.java:834) [?:?]
{code}
> Partition with no column statistics leads to unbalanced calls to
> openTransaction/commitTransaction error during get_partitions_by_names
> ---------------------------------------------------------------------------------------------------------------------------------------
>
> Key: HIVE-23960
> URL: https://issues.apache.org/jira/browse/HIVE-23960
> Project: Hive
> Issue Type: Task
> Reporter: Pravin Sinha
> Assignee: Pravin Sinha
> Priority: Major
>
> {color:#172b4d}Creating a partition with data and adding another partition is
> leading to unbalanced calls to open/commit transaction during
> get_partitions_by_names call.{color}
> {color:#172b4d}Issue was discovered during REPL DUMP operation which uses
> this HMS call to get the metadata of partition. This error occurs when there
> is a partition with no column statistics.{color}
> {color:#172b4d}To reproduce:{color}
> {code:java}
> CREATE TABLE student_part_acid(name string, age int, gpa double) PARTITIONED
> BY (ds string) STORED AS orc;
> LOAD DATA INPATH ‘/user/hive/partDir/student_part_acid/ds=20110924’ INTO
> TABLE student_part_acid partition(ds=20110924);
> ALTER TABLE student_part_acid ADD PARTITION (ds=20110925);
> Now if we try to preform REPL DUMP it fails with this the error "Unbalanced
> calls to open/commit transaction" on the HS2 side.
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
