[
https://issues.apache.org/jira/browse/IMPALA-15335?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Zoltán Borók-Nagy updated IMPALA-15335:
---------------------------------------
Description:
TestAutomaticCatalogInvalidation.test_invalidation_metrics is flaky with
Calcite planner
h3. Error Message
{noformat}
AssertionError: Metric catalog.num-loaded-tables did not reach value 3 in 5s.
Actual value was '0'.
Dumping debug webpages in JSON format...
Dumped memz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/memz.json
Dumped metrics JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/metrics.json
Dumped queries JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/queries.json
Dumped sessions JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/sessions.json
Dumped threadz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/threadz.json
Dumped rpcz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/rpcz.json
Dumping minidumps for impalads/catalogds...
Dumped minidump for Impalad PID 3098167
Dumped minidump for Impalad PID 3098170
Dumped minidump for Impalad PID 3098174
Dumped minidump for Catalogd PID 3098138{noformat}
h3. Stacktrace
{noformat}
custom_cluster/test_automatic_invalidation.py:290: in test_invalidation_metrics
catalogd.wait_for_metric_value(loaded_tables_metric, 3, timeout=5)
baseline_memory_count = 0
baseline_ttl_count = 0
catalogd = <tests.common.impala_service.CatalogdService object at
0x7f4259bd6250>
last_mem_ms = 'catalog.last-memory-pressure-invalidation-ms'
last_mem_tables = 'catalog.last-memory-pressure-invalidated-tables'
last_ttl_ms = 'catalog.last-ttl-invalidation-ms'
last_ttl_ms_1 = 1788749640542
last_ttl_tables = 'catalog.last-ttl-invalidated-tables'
last_ttl_tables_1 = 1
loaded_tables_metric = 'catalog.num-loaded-tables'
max_wait_time = 18
memory_count_after_first = 0
memory_metric = 'catalog.num-memory-pressure-invalidated-tables'
self =
<tests.custom_cluster.test_automatic_invalidation.TestAutomaticCatalogInvalidation
object at 0x7f4300a80750>
ttl_10s = 'catalog.ttl-invalidations-10s'
ttl_1m = 'catalog.ttl-invalidations-01m'
ttl_30m = 'catalog.ttl-invalidations-30m'
ttl_5m = 'catalog.ttl-invalidations-05m'
ttl_count_after_first = 1
ttl_metric = 'catalog.num-ttl-invalidated-tables'
union_query = 'SELECT 1 FROM functional.alltypessmall LIMIT 0 UNION ALL
SELECT 1 FROM functional.alltypesagg LIMIT 0 UNION ALL SELECT 1 FROM
functional.alltypes LIMIT 0'
common/impala_service.py:174: in wait_for_metric_value
self.__metric_timeout_assert(metric_name, expected_value, timeout, value)
allow_greater = False
expected_value = 3
interval = 1
metric_name = 'catalog.num-loaded-tables'
self = <tests.common.impala_service.CatalogdService object at
0x7f4259bd6250>
start_time = 1788749643.715257
timeout = 5
total_wait = 5.0201735496521
value = 0
common/impala_service.py:261: in __metric_timeout_assert
assert 0, assert_string
E AssertionError: Metric catalog.num-loaded-tables did not reach value 3 in
5s. Actual value was '0'.
E Dumping debug webpages in JSON format...
E Dumped memz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/memz.json
E Dumped metrics JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/metrics.json
E Dumped queries JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/queries.json
E Dumped sessions JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/sessions.json
E Dumped threadz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/threadz.json
E Dumped rpcz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/rpcz.json
E Dumping minidumps for impalads/catalogds...
E Dumped minidump for Impalad PID 3098167
E Dumped minidump for Impalad PID 3098170
E Dumped minidump for Impalad PID 3098174
E Dumped minidump for Catalogd PID 3098138{noformat}
was:
TestAutomaticCatalogInvalidation.test_invalidation_metrics is flaky with
Calcite planner
h3. Error Message
AssertionError: Metric catalog.num-loaded-tables did not reach value 3 in 5s.
Actual value was '0'. Dumping debug webpages in JSON format... Dumped memz JSON
to $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/memz.json
Dumped metrics JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/metrics.json
Dumped queries JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/queries.json
Dumped sessions JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/sessions.json
Dumped threadz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/threadz.json
Dumped rpcz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/rpcz.json Dumping
minidumps for impalads/catalogds... Dumped minidump for Impalad PID 3098167
Dumped minidump for Impalad PID 3098170 Dumped minidump for Impalad PID 3098174
Dumped minidump for Catalogd PID 3098138
h3. Stacktrace
custom_cluster/test_automatic_invalidation.py:290: in test_invalidation_metrics
catalogd.wait_for_metric_value(loaded_tables_metric, 3, timeout=5)
baseline_memory_count = 0 baseline_ttl_count = 0 catalogd =
<tests.common.impala_service.CatalogdService object at 0x7f4259bd6250>
last_mem_ms = 'catalog.last-memory-pressure-invalidation-ms' last_mem_tables =
'catalog.last-memory-pressure-invalidated-tables' last_ttl_ms =
'catalog.last-ttl-invalidation-ms' last_ttl_ms_1 = 1788749640542
last_ttl_tables = 'catalog.last-ttl-invalidated-tables' last_ttl_tables_1 = 1
loaded_tables_metric = 'catalog.num-loaded-tables' max_wait_time = 18
memory_count_after_first = 0 memory_metric =
'catalog.num-memory-pressure-invalidated-tables' self =
<tests.custom_cluster.test_automatic_invalidation.TestAutomaticCatalogInvalidation
object at 0x7f4300a80750> ttl_10s = 'catalog.ttl-invalidations-10s' ttl_1m =
'catalog.ttl-invalidations-01m' ttl_30m = 'catalog.ttl-invalidations-30m'
ttl_5m = 'catalog.ttl-invalidations-05m' ttl_count_after_first = 1 ttl_metric =
'catalog.num-ttl-invalidated-tables' union_query = 'SELECT 1 FROM
functional.alltypessmall LIMIT 0 UNION ALL SELECT 1 FROM functional.alltypesagg
LIMIT 0 UNION ALL SELECT 1 FROM functional.alltypes LIMIT 0'
common/impala_service.py:174: in wait_for_metric_value
self.__metric_timeout_assert(metric_name, expected_value, timeout, value)
allow_greater = False expected_value = 3 interval = 1 metric_name =
'catalog.num-loaded-tables' self = <tests.common.impala_service.CatalogdService
object at 0x7f4259bd6250> start_time = 1788749643.715257 timeout = 5 total_wait
= 5.0201735496521 value = 0 common/impala_service.py:261: in
__metric_timeout_assert assert 0, assert_string E AssertionError: Metric
catalog.num-loaded-tables did not reach value 3 in 5s. Actual value was '0'. E
Dumping debug webpages in JSON format... E Dumped memz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/memz.json E
Dumped metrics JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/metrics.json E
Dumped queries JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/queries.json E
Dumped sessions JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/sessions.json E
Dumped threadz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/threadz.json E
Dumped rpcz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/rpcz.json E
Dumping minidumps for impalads/catalogds... E Dumped minidump for Impalad PID
3098167 E Dumped minidump for Impalad PID 3098170 E Dumped minidump for Impalad
PID 3098174 E Dumped minidump for Catalogd PID 3098138 actual_value = 0
assert_string = "Metric catalog.num-loaded-tables did not reach value 3 in 5s.
Actual value was '0'.\nDumping debug webpages in JSON f... minidump for Impalad
PID 3098170\nDumped minidump for Impalad PID 3098174\nDumped minidump for
Catalogd PID 3098138\n" catalogd_pids = ['3098138'] datetime_string =
'20260906_19:54:08' debug_page = 'rpcz' debug_pages = ['memz', 'metrics',
'queries', 'sessions', 'threadz', 'rpcz'] diag_dir =
'/data/jenkins/workspace/impala-asf-master-core-calcite/repos/Impala/logs/metric_timeout_diags_20260906_19:54:08'
expected_value = 3 impala_home =
'/data/jenkins/workspace/impala-asf-master-core-calcite/repos/Impala'
impalad_pids = ['3098167', '3098170', '3098174'] json_diag_string = 'Dumping
debug webpages in JSON format...\nDumped memz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:0...54:08/json/threadz.json\nDumped
rpcz JSON to
$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/rpcz.json\n'
json_dir =
'/data/jenkins/workspace/impala-asf-master-core-calcite/repos/Impala/logs/metric_timeout_diags_20260906_19:54:08/json'
json_filename =
'/data/jenkins/workspace/impala-asf-master-core-calcite/repos/Impala/logs/metric_timeout_diags_20260906_19:54:08/json/rpcz.json'
json_filename_rewritten =
'$IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/rpcz.json'
log_dir =
'/data/jenkins/workspace/impala-asf-master-core-calcite/repos/Impala/logs'
metric_name = 'catalog.num-loaded-tables' minidump_diag_string = 'Dumping
minidumps for impalads/catalogds...\nDumped minidump for Impalad PID
3098167\nDumped minidump for Impalad PID 3098170\nDumped minidump for Impalad
PID 3098174\nDumped minidump for Catalogd PID 3098138\n' pid = '3098138' self =
<tests.common.impala_service.CatalogdService object at 0x7f4259bd6250> timeout
= 5
> TestAutomaticCatalogInvalidation.test_invalidation_metrics is flaky with
> Calcite planner
> ----------------------------------------------------------------------------------------
>
> Key: IMPALA-15335
> URL: https://issues.apache.org/jira/browse/IMPALA-15335
> Project: IMPALA
> Issue Type: Bug
> Reporter: Zoltán Borók-Nagy
> Priority: Major
> Labels: broken-build
>
> TestAutomaticCatalogInvalidation.test_invalidation_metrics is flaky with
> Calcite planner
> h3. Error Message
> {noformat}
> AssertionError: Metric catalog.num-loaded-tables did not reach value 3 in 5s.
> Actual value was '0'.
> Dumping debug webpages in JSON format...
> Dumped memz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/memz.json
> Dumped metrics JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/metrics.json
> Dumped queries JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/queries.json
> Dumped sessions JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/sessions.json
> Dumped threadz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/threadz.json
> Dumped rpcz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/rpcz.json
> Dumping minidumps for impalads/catalogds...
> Dumped minidump for Impalad PID 3098167
> Dumped minidump for Impalad PID 3098170
> Dumped minidump for Impalad PID 3098174
> Dumped minidump for Catalogd PID 3098138{noformat}
> h3. Stacktrace
> {noformat}
> custom_cluster/test_automatic_invalidation.py:290: in
> test_invalidation_metrics
> catalogd.wait_for_metric_value(loaded_tables_metric, 3, timeout=5)
> baseline_memory_count = 0
> baseline_ttl_count = 0
> catalogd = <tests.common.impala_service.CatalogdService object at
> 0x7f4259bd6250>
> last_mem_ms = 'catalog.last-memory-pressure-invalidation-ms'
> last_mem_tables = 'catalog.last-memory-pressure-invalidated-tables'
> last_ttl_ms = 'catalog.last-ttl-invalidation-ms'
> last_ttl_ms_1 = 1788749640542
> last_ttl_tables = 'catalog.last-ttl-invalidated-tables'
> last_ttl_tables_1 = 1
> loaded_tables_metric = 'catalog.num-loaded-tables'
> max_wait_time = 18
> memory_count_after_first = 0
> memory_metric = 'catalog.num-memory-pressure-invalidated-tables'
> self =
> <tests.custom_cluster.test_automatic_invalidation.TestAutomaticCatalogInvalidation
> object at 0x7f4300a80750>
> ttl_10s = 'catalog.ttl-invalidations-10s'
> ttl_1m = 'catalog.ttl-invalidations-01m'
> ttl_30m = 'catalog.ttl-invalidations-30m'
> ttl_5m = 'catalog.ttl-invalidations-05m'
> ttl_count_after_first = 1
> ttl_metric = 'catalog.num-ttl-invalidated-tables'
> union_query = 'SELECT 1 FROM functional.alltypessmall LIMIT 0 UNION
> ALL SELECT 1 FROM functional.alltypesagg LIMIT 0 UNION ALL SELECT 1 FROM
> functional.alltypes LIMIT 0'
> common/impala_service.py:174: in wait_for_metric_value
> self.__metric_timeout_assert(metric_name, expected_value, timeout, value)
> allow_greater = False
> expected_value = 3
> interval = 1
> metric_name = 'catalog.num-loaded-tables'
> self = <tests.common.impala_service.CatalogdService object at
> 0x7f4259bd6250>
> start_time = 1788749643.715257
> timeout = 5
> total_wait = 5.0201735496521
> value = 0
> common/impala_service.py:261: in __metric_timeout_assert
> assert 0, assert_string
> E AssertionError: Metric catalog.num-loaded-tables did not reach value 3 in
> 5s. Actual value was '0'.
> E Dumping debug webpages in JSON format...
> E Dumped memz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/memz.json
> E Dumped metrics JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/metrics.json
> E Dumped queries JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/queries.json
> E Dumped sessions JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/sessions.json
> E Dumped threadz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/threadz.json
> E Dumped rpcz JSON to
> $IMPALA_HOME/logs/metric_timeout_diags_20260906_19:54:08/json/rpcz.json
> E Dumping minidumps for impalads/catalogds...
> E Dumped minidump for Impalad PID 3098167
> E Dumped minidump for Impalad PID 3098170
> E Dumped minidump for Impalad PID 3098174
> E Dumped minidump for Catalogd PID 3098138{noformat}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
