[
https://issues.apache.org/jira/browse/IMPALA-14696?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Dániel Gábor Vankó updated IMPALA-14696:
----------------------------------------
Labels: broken-build flaky flaky-test (was: broken-build flaky-test)
> TestSessionExpiration.test_closing_idle_connection is flaky
> -----------------------------------------------------------
>
> Key: IMPALA-14696
> URL: https://issues.apache.org/jira/browse/IMPALA-14696
> Project: IMPALA
> Issue Type: Bug
> Reporter: Dániel Gábor Vankó
> Assignee: Dániel Gábor Vankó
> Priority: Major
> Labels: broken-build, flaky, flaky-test
>
> Saw in some test that test_closing_idle_connection fails with the following:
> h2. Error Message
> {noformat}
> AssertionError: Metric
> impala.thrift-server.hiveserver2-frontend.connections-in-use did not reach
> value 0 in 5s. Actual value was '1'.
> Dumping debug webpages in JSON format...
> Dumped memz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/memz.json
> Dumped metrics JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/metrics.json
> Dumped queries JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/queries.json
> Dumped sessions JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/sessions.json
> Dumped threadz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/threadz.json
> Dumped rpcz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json
> Dumping minidumps for impalads/catalogds...
> Dumped minidump for Impalad PID 250362
> Dumped minidump for Catalogd PID 250268{noformat}
> h2.
> Stacktrace
> {noformat}
> custom_cluster/test_session_expiration.py:171: in test_closing_idle_connection
> impalad.service.wait_for_metric_value(num_connections_metrics_name,
> client = <tests.common.impala_connection.ImpylaHS2Connection
> object at 0xffff079248b0>
> impalad = <ImpaladProcess PID: 250362
> (/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/be/build/latest/serv..._io_threads=1
> -num_s3_io_threads=1 -num_s3_file_oper_io_threads=1 -num_sfs_io_threads=1
> -geospatial_library=HIVE_ESRI)>
> num_connections = 0
> num_connections_metrics_name =
> 'impala.thrift-server.hiveserver2-frontend.connections-in-use'
> num_expired = 1
> protocol = 'hiveserver2'
> self =
> <tests.custom_cluster.test_session_expiration.TestSessionExpiration object at
> 0xffff072ad760>
> common/impala_service.py:164: in wait_for_metric_value
> self.__metric_timeout_assert(metric_name, expected_value, timeout, value)
> allow_greater = False
> expected_value = 0
> interval = 1
> metric_name =
> 'impala.thrift-server.hiveserver2-frontend.connections-in-use'
> self = <tests.common.impala_service.ImpaladService object at
> 0xffff0792ae20>
> start_time = 1769023935.4829047
> timeout = 5
> total_wait = 5.0343897342681885
> value = 1
> common/impala_service.py:251: in __metric_timeout_assert
> assert 0, assert_string
> E AssertionError: Metric
> impala.thrift-server.hiveserver2-frontend.connections-in-use did not reach
> value 0 in 5s. Actual value was '1'.
> E Dumping debug webpages in JSON format...
> E Dumped memz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/memz.json
> E Dumped metrics JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/metrics.json
> E Dumped queries JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/queries.json
> E Dumped sessions JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/sessions.json
> E Dumped threadz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/threadz.json
> E Dumped rpcz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json
> E Dumping minidumps for impalads/catalogds...
> E Dumped minidump for Impalad PID 250362
> E Dumped minidump for Catalogd PID 250268
> actual_value = 1
> assert_string = "Metric
> impala.thrift-server.hiveserver2-frontend.connections-in-use did not reach
> value 0 in 5s. Actual value was '1'...minidumps for
> impalads/catalogds...\nDumped minidump for Impalad PID 250362\nDumped
> minidump for Catalogd PID 250268\n"
> catalogd_pids = ['250268']
> datetime_string = '20260121_11:32:20'
> debug_page = 'rpcz'
> debug_pages = ['memz', 'metrics', 'queries', 'sessions', 'threadz',
> 'rpcz']
> diag_dir =
> '/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/metric_timeout_diags_20260121_11:32:20'
> expected_value = 0
> impala_home =
> '/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala'
> impalad_pids = ['250362']
> json_diag_string = 'Dumping debug webpages in JSON format...\nDumped
> memz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:2...32:20/json/threadz.json\nDumped
> rpcz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json\n'
> json_dir =
> '/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/metric_timeout_diags_20260121_11:32:20/json'
> json_filename =
> '/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json'
> json_filename_rewritten =
> '$IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json'
> log_dir =
> '/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs'
> metric_name =
> 'impala.thrift-server.hiveserver2-frontend.connections-in-use'
> minidump_diag_string = 'Dumping minidumps for
> impalads/catalogds...\nDumped minidump for Impalad PID 250362\nDumped
> minidump for Catalogd PID 250268\n'
> pid = '250268'
> self = <tests.common.impala_service.ImpaladService object at
> 0xffff0792ae20>
> timeout = 5{noformat}
> h2. Standard Error
> {noformat}
> --------------------------------- Captured Err
> ---------------------------------
> -- 2026-01-21 11:31:44,397 INFO MainThread: Starting cluster with
> command:
> /data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/bin/start-impala-cluster.py
> '--state_store_args=--statestore_update_frequency_ms=50
> --statestore_priority_update_frequency_ms=50
> --statestore_heartbeat_frequency_ms=50' --cluster_size=1 --num_coordinators=1
> --log_dir=/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/custom_cluster_tests/TestSessionExpiration/test_closing_idle_connection
> --log_level=1 '--impalad_args=--idle_session_timeout=10
> --idle_client_poll_period_s=1 ' --impalad_args=--default_query_options=
> 11:31:44 MainThread: Found 0 impalad/0 statestored/0 catalogd process(es)
> 11:31:44 MainThread: Starting State Store logging to
> /data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/custom_cluster_tests/TestSessionExpiration/test_closing_idle_connection/statestored.INFO
> 11:31:45 MainThread: Starting Catalog Service logging to
> /data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/custom_cluster_tests/TestSessionExpiration/test_closing_idle_connection/catalogd.INFO
> 11:31:47 MainThread: Starting Impala Daemon logging to
> /data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/custom_cluster_tests/TestSessionExpiration/test_closing_idle_connection/impalad.INFO
> 11:31:49 MainThread: Found 1 impalad/1 statestored/1 catalogd process(es)
> 11:31:49 MainThread: Waiting for Impalad webserver port 25000
> 11:31:49 MainThread: Waiting for Impalad webserver port 25000
> 11:31:50 MainThread: Waiting for Impalad webserver port 25000
> 11:31:50 MainThread: Waiting for Impalad webserver port 25000
> 11:31:52 MainThread: Waiting for coordinator client services - hs2 port:
> 21050 hs2-http port: 28000 beeswax port: 21000
> 11:31:52 MainThread: Getting num_known_live_backends from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> 11:31:52 MainThread: num_known_live_backends has reached value: 1
> 11:31:52 MainThread: Total wait: 3.65s
> 11:31:52 MainThread: Actual log file names:
> 11:31:52 MainThread: statestored.INFO ->
> statestored.impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com.jenkins.log.INFO.20260121-113144.250206
> 11:31:52 MainThread: catalogd.INFO ->
> catalogd.impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com.jenkins.log.INFO.20260121-113146.250268
> 11:31:52 MainThread: impalad.INFO ->
> impalad.impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com.jenkins.log.INFO.20260121-113147.250362
> 11:31:52 MainThread: Impala Cluster Running with 1 nodes (1 coordinators, 1
> executors).
> -- 2026-01-21 11:31:52,772 DEBUG MainThread: Found 1 impalad/1
> statestored/1 catalogd process(es)
> -- 2026-01-21 11:31:52,772 INFO MainThread: Getting metric:
> statestore.live-backends from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25010
> -- 2026-01-21 11:31:52,778 INFO MainThread: Metric
> 'statestore.live-backends' has reached desired value: 2. total_wait: 0s
> -- 2026-01-21 11:31:52,778 DEBUG MainThread: Getting
> num_known_live_backends from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:52,782 INFO MainThread: num_known_live_backends has
> reached value: 1
> -- 2026-01-21 11:31:52,783 INFO MainThread: beeswax:
> set
> client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:31:52,783 INFO MainThread: beeswax: connected to
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21000 with
> beeswax
> -- 2026-01-21 11:31:52,783 INFO MainThread: hs2:
> set
> client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:31:52,783 INFO MainThread: hs2: connected to
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21050 with
> impyla hs2
> -- 2026-01-21 11:31:52,783 INFO MainThread: hs2-http:
> set
> client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:31:52,785 INFO MainThread: hs2-http: connected to
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:28000 with
> impyla hs2-http
> -- 2026-01-21 11:31:52,785 INFO MainThread: hs2-feng:
> set
> client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:31:52,785 INFO MainThread: hs2-feng: connected to
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:11050 with
> impyla hs2-feng
> -- 2026-01-21 11:31:52,788 INFO MainThread: beeswax: closing beeswax
> connection to:
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21000
> -- 2026-01-21 11:31:52,788 INFO MainThread: hs2: closing 1 sync and 0
> async hs2 connections to:
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21050
> -- 2026-01-21 11:31:52,788 INFO MainThread: hs2-http: closing 1 sync and
> 0 async hs2-http connections to:
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:28000
> -- 2026-01-21 11:31:52,788 INFO MainThread: hs2-feng: closing 1 sync and
> 0 async hs2-feng connections to:
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:11050
> -- 2026-01-21 11:31:52,809 WARNING MainThread: beeswax protocol is
> deprecated.
> -- 2026-01-21 11:31:52,809 INFO MainThread: beeswax:
> set
> client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:31:52,809 INFO MainThread: beeswax: connected to
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21000 with
> beeswax
> -- 2026-01-21 11:31:52,809 INFO MainThread: beeswax: executing against
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21000
> select 1;
> -- 2026-01-21 11:31:53,191 INFO MainThread: Started query
> df4d8a404151a8b0:a0da1a1000000000
> -- 2026-01-21 11:31:53,247 INFO MainThread: Getting metric:
> impala.thrift-server.beeswax-frontend.connections-in-use from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:53,253 INFO MainThread: Metric
> 'impala.thrift-server.beeswax-frontend.connections-in-use' has reached
> desired value: 1. total_wait: 0s
> -- 2026-01-21 11:31:53,254 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:53,259 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 0s
> -- 2026-01-21 11:31:53,259 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:54,260 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:54,266 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait:
> 1.0065803527832031s
> -- 2026-01-21 11:31:54,267 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:55,267 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:55,273 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait:
> 2.0133278369903564s
> -- 2026-01-21 11:31:55,273 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:56,274 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:56,280 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait:
> 3.0207297801971436s
> -- 2026-01-21 11:31:56,281 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:57,282 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:57,288 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait:
> 4.028151035308838s
> -- 2026-01-21 11:31:57,288 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:58,289 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:58,295 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait:
> 5.03555703163147s
> -- 2026-01-21 11:31:58,295 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:59,296 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:59,302 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait:
> 6.0421106815338135s
> -- 2026-01-21 11:31:59,302 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:00,303 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:00,309 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait:
> 7.049506664276123s
> -- 2026-01-21 11:32:00,309 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:01,310 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:01,317 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait:
> 8.056854486465454s
> -- 2026-01-21 11:32:01,317 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:02,318 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:02,324 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait:
> 9.06404423713684s
> -- 2026-01-21 11:32:02,324 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:03,325 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:03,331 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait:
> 10.07157278060913s
> -- 2026-01-21 11:32:03,331 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:04,333 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:04,339 INFO MainThread: Metric
> 'impala-server.num-sessions-expired' has reached desired value: 1.
> total_wait: 11.078944683074951s
> -- 2026-01-21 11:32:04,339 INFO MainThread: Getting metric:
> impala.thrift-server.beeswax-frontend.connections-in-use from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:04,344 INFO MainThread: Metric
> 'impala.thrift-server.beeswax-frontend.connections-in-use' has reached
> desired value: 0. total_wait: 0s
> -- 2026-01-21 11:32:04,344 INFO MainThread: beeswax: closing beeswax
> connection to:
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21000
> -- 2026-01-21 11:32:04,354 INFO MainThread: hs2:
> set
> client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:32:04,355 INFO MainThread: hs2: connected to
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21050 with
> impyla hs2
> -- 2026-01-21 11:32:04,356 INFO MainThread: hs2: executing against Impala
> at impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21050.
> session: 204b4c0587e2d9c0:0c9b2506804d61ac main_cursor: True user: None
> select 1;
> -- 2026-01-21 11:32:04,383 INFO MainThread:
> 5d43097c66b5e745:cad5462c00000000: query started
> -- 2026-01-21 11:32:04,386 INFO MainThread:
> 5d43097c66b5e745:cad5462c00000000: getting log for operation
> -- 2026-01-21 11:32:04,387 INFO MainThread:
> 5d43097c66b5e745:cad5462c00000000: getting runtime profile operation
> -- 2026-01-21 11:32:04,388 INFO MainThread:
> 5d43097c66b5e745:cad5462c00000000: closing query for operation
> -- 2026-01-21 11:32:04,389 INFO MainThread: Getting metric:
> impala.thrift-server.hiveserver2-frontend.connections-in-use from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:04,394 INFO MainThread: Could not connect to ('::1',
> 21050, 0, 0)
> Traceback (most recent call last):
> File
> "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/urllib3/connectionpool.py",
> line 377, in _make_request
> httplib_response = conn.getresponse(buffering=True)
> TypeError: getresponse() got an unexpected keyword argument 'buffering'
> During handling of the above exception, another exception occurred:
> Traceback (most recent call last):
> File
> "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/thrift/transport/TSocket.py",
> line 137, in open
> handle.connect(sockaddr)
> ConnectionRefusedError: [Errno 111] Connection refused
> -- 2026-01-21 11:32:04,397 INFO MainThread: Metric
> 'impala.thrift-server.hiveserver2-frontend.connections-in-use' has reached
> desired value: 1. total_wait: 0s
> -- 2026-01-21 11:32:04,397 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:04,402 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 0s
> -- 2026-01-21 11:32:04,402 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:05,403 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:05,409 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait:
> 1.0054309368133545s
> -- 2026-01-21 11:32:05,409 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:06,410 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:06,416 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait:
> 2.0127756595611572s
> -- 2026-01-21 11:32:06,416 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:07,417 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:07,423 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait:
> 3.0200905799865723s
> -- 2026-01-21 11:32:07,423 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:08,425 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:08,431 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait:
> 4.027472257614136s
> -- 2026-01-21 11:32:08,431 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:09,432 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:09,438 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait:
> 5.034852981567383s
> -- 2026-01-21 11:32:09,438 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:10,439 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:10,445 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait:
> 6.042134046554565s
> -- 2026-01-21 11:32:10,445 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:11,447 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:11,453 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait:
> 7.049464464187622s
> -- 2026-01-21 11:32:11,453 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:12,454 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:12,460 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait:
> 8.057080030441284s
> -- 2026-01-21 11:32:12,460 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:13,462 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:13,468 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait:
> 9.064432859420776s
> -- 2026-01-21 11:32:13,468 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:14,469 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:14,475 INFO MainThread: Waiting for metric value
> 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait:
> 10.071696281433105s
> -- 2026-01-21 11:32:14,475 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:15,476 INFO MainThread: Getting metric:
> impala-server.num-sessions-expired from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:15,482 INFO MainThread: Metric
> 'impala-server.num-sessions-expired' has reached desired value: 2.
> total_wait: 11.079020500183105s
> -- 2026-01-21 11:32:15,482 INFO MainThread: Getting metric:
> impala.thrift-server.hiveserver2-frontend.connections-in-use from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:15,488 INFO MainThread: Waiting for metric value
> 'impala.thrift-server.hiveserver2-frontend.connections-in-use'=0. Current
> value: 1. total_wait: 0s
> -- 2026-01-21 11:32:15,488 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:16,489 INFO MainThread: Getting metric:
> impala.thrift-server.hiveserver2-frontend.connections-in-use from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:16,495 INFO MainThread: Waiting for metric value
> 'impala.thrift-server.hiveserver2-frontend.connections-in-use'=0. Current
> value: 1. total_wait: 1.006498098373413s
> -- 2026-01-21 11:32:16,495 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:17,496 INFO MainThread: Getting metric:
> impala.thrift-server.hiveserver2-frontend.connections-in-use from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:17,502 INFO MainThread: Waiting for metric value
> 'impala.thrift-server.hiveserver2-frontend.connections-in-use'=0. Current
> value: 1. total_wait: 2.0138399600982666s
> -- 2026-01-21 11:32:17,502 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:18,503 INFO MainThread: Getting metric:
> impala.thrift-server.hiveserver2-frontend.connections-in-use from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:18,509 INFO MainThread: Waiting for metric value
> 'impala.thrift-server.hiveserver2-frontend.connections-in-use'=0. Current
> value: 1. total_wait: 3.0201923847198486s
> -- 2026-01-21 11:32:18,509 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:19,509 INFO MainThread: Getting metric:
> impala.thrift-server.hiveserver2-frontend.connections-in-use from
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:19,516 INFO MainThread: Waiting for metric value
> 'impala.thrift-server.hiveserver2-frontend.connections-in-use'=0. Current
> value: 1. total_wait: 4.02702784538269s
> -- 2026-01-21 11:32:19,516 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:20,517 INFO MainThread: Metric
> impala.thrift-server.hiveserver2-frontend.connections-in-use did not reach
> value 0 in 5s. Actual value was '1'. total_wait: 5.0343897342681885s.
> Failing...
> -- 2026-01-21 11:32:50,693 INFO MainThread: hs2: closing 1 sync and 0
> async hs2 connections to:
> impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21050
> -- 2026-01-21 11:32:50,694 ERROR MainThread: Failed to open transport
> (tries_left=3)
> Traceback (most recent call last):
> File
> "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/tests/custom_cluster/test_session_expiration.py",
> line 171, in test_closing_idle_connection
> impalad.service.wait_for_metric_value(num_connections_metrics_name,
> File
> "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/tests/common/impala_service.py",
> line 164, in wait_for_metric_value
> self.__metric_timeout_assert(metric_name, expected_value, timeout, value)
> File
> "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/tests/common/impala_service.py",
> line 251, in __metric_timeout_assert
> assert 0, assert_string
> AssertionError: Metric
> impala.thrift-server.hiveserver2-frontend.connections-in-use did not reach
> value 0 in 5s. Actual value was '1'.
> Dumping debug webpages in JSON format...
> Dumped memz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/memz.json
> Dumped metrics JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/metrics.json
> Dumped queries JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/queries.json
> Dumped sessions JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/sessions.json
> Dumped threadz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/threadz.json
> Dumped rpcz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json
> Dumping minidumps for impalads/catalogds...
> Dumped minidump for Impalad PID 250362
> Dumped minidump for Catalogd PID 250268
> During handling of the above exception, another exception occurred:
> Traceback (most recent call last):
> File
> "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/impala/hiveserver2.py",
> line 1195, in _execute
> open_transport(transport)
> File
> "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/impala/hiveserver2.py",
> line 1259, in open_transport
> transport.open()
> File
> "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/thrift/transport/TTransport.py",
> line 155, in open
> return self.__trans.open()
> File
> "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/thrift/transport/TSocket.py",
> line 121, in open
> raise TTransportException(type=TTransportException.ALREADY_OPEN,
> message="already open")
> thrift.transport.TTransport.TTransportException: already open{noformat}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
