[
https://issues.apache.org/jira/browse/IMPALA-13418?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17887167#comment-17887167
]
Quanlong Huang commented on IMPALA-13418:
-----------------------------------------
In HMS logs, it seems the underlying DB (postgresql) is not responsive in 30s:
{noformat}
2024-10-01T11:35:22,454 WARN [TThreadPoolServer WorkerProcess-135]
metastore.ObjectStore: Failed to get database hive.default, returning
NoSuchObjectException
org.apache.hadoop.hive.metastore.api.MetaException: objectstore - Connection is
not available, request timed out after 30000ms.
at
org.apache.hadoop.hive.metastore.ObjectStore$GetHelper.run(ObjectStore.java:4339)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at
org.apache.hadoop.hive.metastore.ObjectStore.getDatabaseInternal(ObjectStore.java:902)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at
org.apache.hadoop.hive.metastore.ObjectStore.getDatabase(ObjectStore.java:874)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_261]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_261]
at
org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:97)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at com.sun.proxy.$Proxy33.getDatabase(Unknown Source) ~[?:?]
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_database_req(HiveMetaStore.java:1627)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_261]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_261]
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:160)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:121)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at com.sun.proxy.$Proxy34.get_database_req(Unknown Source) ~[?:?]
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$get_database_req.getResult(ThriftHiveMetastore.java:18802)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$get_database_req.getResult(ThriftHiveMetastore.java:18781)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38)
~[libthrift-0.16.0.jar:0.16.0]
at
org.apache.hadoop.hive.metastore.TUGIBasedProcessor$1.run(TUGIBasedProcessor.java:111)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at
org.apache.hadoop.hive.metastore.TUGIBasedProcessor$1.run(TUGIBasedProcessor.java:107)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at java.security.AccessController.doPrivileged(Native Method)
~[?:1.8.0_261]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_261]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)
~[hadoop-common-3.1.1.7.3.0.0-128.jar:?]
at
org.apache.hadoop.hive.metastore.TUGIBasedProcessor.process(TUGIBasedProcessor.java:119)
~[hive-standalone-metastore-3.1.3000.7.3.0.0-128.jar:3.1.3000.7.3.0.0-128]
at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:250)
~[libthrift-0.16.0.jar:0.16.0]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_261]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_261]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_261] {noformat}
> test_read_hive_inserts failed to open new Hive session
> ------------------------------------------------------
>
> Key: IMPALA-13418
> URL: https://issues.apache.org/jira/browse/IMPALA-13418
> Project: IMPALA
> Issue Type: Bug
> Reporter: Quanlong Huang
> Assignee: Quanlong Huang
> Priority: Critical
>
> Stacktrace
> {noformat}
> /data/jenkins/workspace/impala-asf-master-exhaustive-data-cache/repos/Impala/tests/stress/test_acid_stress.py:172:
> in test_read_hive_inserts
> self._run_test_read_hive_inserts(unique_database, is_partitioned)
> /data/jenkins/workspace/impala-asf-master-exhaustive-data-cache/repos/Impala/tests/stress/test_acid_stress.py:150:
> in _run_test_read_hive_inserts
> sleep_seconds=3)])
> stress/stress_util.py:46: in run_tasks
> pool.map_async(Task.run, tasks).get(timeout_seconds)
> /data/jenkins/workspace/impala-asf-master-exhaustive-data-cache/Impala-Toolchain/toolchain-packages-gcc10.4.0/python-2.7.16/lib/python2.7/multiprocessing/pool.py:572:
> in get
> raise self._value
> E RuntimeError: Picked up JAVA_TOOL_OPTIONS:
> -javaagent:/data/jenkins/workspace/impala-asf-master-exhaustive-data-cache/repos/Impala/fe/target/dependency/jamm-0.4.0.jar
> E SLF4J: Class path contains multiple SLF4J bindings.
> E SLF4J: Found binding in
> [jar:file:/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/Impala-Toolchain/cdp_components-49623641/apache-hive-3.1.3000.7.3.0.0-128-bin/lib/log4j-slf4j-impl-2.18.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> E SLF4J: Found binding in
> [jar:file:/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/Impala-Toolchain/cdp_components-49623641/hadoop-3.1.1.7.3.0.0-128/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> E SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an
> explanation.
> E SLF4J: Actual binding is of type
> [org.apache.logging.slf4j.Log4jLoggerFactory]
> E Picked up JAVA_TOOL_OPTIONS:
> -javaagent:/data/jenkins/workspace/impala-asf-master-exhaustive-data-cache/repos/Impala/fe/target/dependency/jamm-0.4.0.jar
> E SLF4J: Class path contains multiple SLF4J bindings.
> E SLF4J: Found binding in
> [jar:file:/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/Impala-Toolchain/cdp_components-49623641/apache-hive-3.1.3000.7.3.0.0-128-bin/lib/log4j-slf4j-impl-2.18.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> E SLF4J: Found binding in
> [jar:file:/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/Impala-Toolchain/cdp_components-49623641/hadoop-3.1.1.7.3.0.0-128/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> E SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an
> explanation.
> E SLF4J: Actual binding is of type
> [org.apache.logging.slf4j.Log4jLoggerFactory]
> E 24/10/01 11:35:22 [main]: WARN jdbc.HiveConnection: Failed to connect to
> localhost:11050
> E Error: Could not open client transport with JDBC Uri:
> jdbc:hive2://localhost:11050: Failed to open new session: Database default
> does not exist (state=08S01,code=0){noformat}
> The stacktrace in stderr:
> {noformat}
> refresh test_read_hive_inserts_8bb30aca.test_read_hive_inserts;
> -- 2024-10-01 11:30:23,646 INFO Thread-1: Started query
> 154524f9b3e6e4f3:7e3ed1fa00000000
> Traceback (most recent call last):
> File
> "/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/repos/Impala/tests/stress/stress_util.py",
> line 36, in run
> return self.func(*self.args, **self.kwargs)
> File
> "/data/jenkins/workspace/impala-asf-master-exhaustive-data-cache/repos/Impala/tests/stress/test_acid_stress.py",
> line 87, in _hive_role_write_inserts
> self.run_stmt_in_hive(OVERWRITE_SQL)
> File
> "/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/repos/Impala/tests/common/impala_test_suite.py",
> line 1083, in run_stmt_in_hive
> return cls.run_stmt_in_beeline(url, username, stmt)
> File
> "/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/repos/Impala/tests/common/impala_test_suite.py",
> line 1118, in run_stmt_in_beeline
> raise RuntimeError(stderr)
> RuntimeError: Picked up JAVA_TOOL_OPTIONS:
> -javaagent:/data/jenkins/workspace/impala-asf-master-exhaustive-data-cache/repos/Impala/fe/target/dependency/jamm-0.4.0.jar
> SLF4J: Class path contains multiple SLF4J bindings.
> SLF4J: Found binding in
> [jar:file:/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/Impala-Toolchain/cdp_components-49623641/apache-hive-3.1.3000.7.3.0.0-128-bin/lib/log4j-slf4j-impl-2.18.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: Found binding in
> [jar:file:/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/Impala-Toolchain/cdp_components-49623641/hadoop-3.1.1.7.3.0.0-128/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an
> explanation.
> SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
> Picked up JAVA_TOOL_OPTIONS:
> -javaagent:/data/jenkins/workspace/impala-asf-master-exhaustive-data-cache/repos/Impala/fe/target/dependency/jamm-0.4.0.jar
> SLF4J: Class path contains multiple SLF4J bindings.
> SLF4J: Found binding in
> [jar:file:/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/Impala-Toolchain/cdp_components-49623641/apache-hive-3.1.3000.7.3.0.0-128-bin/lib/log4j-slf4j-impl-2.18.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: Found binding in
> [jar:file:/data0/jenkins/workspace/impala-asf-master-exhaustive-data-cache/Impala-Toolchain/cdp_components-49623641/hadoop-3.1.1.7.3.0.0-128/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an
> explanation.
> SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
> 24/10/01 11:35:22 [main]: WARN jdbc.HiveConnection: Failed to connect to
> localhost:11050
> Error: Could not open client transport with JDBC Uri:
> jdbc:hive2://localhost:11050: Failed to open new session: Database default
> does not exist (state=08S01,code=0){noformat}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
