[
https://issues.apache.org/jira/browse/IMPALA-13280?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Andrew Sherman updated IMPALA-13280:
------------------------------------
Description:
Recent changes seem to have resulted in memory oversubscription errors in
TestQuery tests.
Error Message
{code}
query_test/test_queries.py:146: in test_except
self.run_test_case('QueryTest/except', vector) common/impala_test_suite.py:789:
in run_test_case user=test_section.get('USER', '').strip() or None)
common/impala_test_suite.py:708: in __exec_in_impala result =
self.__execute_query(target_impalad_client, query, user=user)
common/impala_test_suite.py:1079: in __execute_query return
impalad_client.execute(query, user=user) common/impala_connection.py:378: in
execute self.__cursor.execute(sql_stmt, configuration=self.__query_options)
/data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:343:
in execute self._wait_to_finish() # make execute synchronous
/data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:428:
in _wait_to_finish raise OperationalError(resp.errorMessage) E
OperationalError: Failed to get minimum memory reservation of 4.09 MB on daemon
impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000 for query
8e498fe323d4f770:0364fa4900000000 due to following error: Memory limit
exceeded: Could not allocate memory while trying to increase reservation. E
Query(8e498fe323d4f770:0364fa4900000000) could not allocate 4.09 MB without
exceeding limit. E Error occurred on backend
impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000 E Memory
left in process limit: 1.19 GB E Query(8e498fe323d4f770:0364fa4900000000):
Reservation=0 ReservationLimit=9.60 GB OtherMemory=0 Total=0 Peak=0 E Memory
is likely oversubscribed. Reducing query concurrency or configuring admission
control may help avoid this error.
{code}
Stacktrace
{code}
query_test/test_queries.py:146: in test_except
self.run_test_case('QueryTest/except', vector)
common/impala_test_suite.py:789: in run_test_case
user=test_section.get('USER', '').strip() or None)
common/impala_test_suite.py:708: in __exec_in_impala
result = self.__execute_query(target_impalad_client, query, user=user)
common/impala_test_suite.py:1079: in __execute_query
return impalad_client.execute(query, user=user)
common/impala_connection.py:378: in execute
self.__cursor.execute(sql_stmt, configuration=self.__query_options)
/data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:343:
in execute
self._wait_to_finish() # make execute synchronous
/data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:428:
in _wait_to_finish
raise OperationalError(resp.errorMessage)
E OperationalError: Failed to get minimum memory reservation of 4.09 MB on
daemon impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000 for
query 8e498fe323d4f770:0364fa4900000000 due to following error: Memory limit
exceeded: Could not allocate memory while trying to increase reservation.
E Query(8e498fe323d4f770:0364fa4900000000) could not allocate 4.09 MB without
exceeding limit.
E Error occurred on backend
impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000
E Memory left in process limit: 1.19 GB
E Query(8e498fe323d4f770:0364fa4900000000): Reservation=0
ReservationLimit=9.60 GB OtherMemory=0 Total=0 Peak=0
E Memory is likely oversubscribed. Reducing query concurrency or configuring
admission control may help avoid this error.
{code}
Standard Error
{code}
-- Executing use functional_parquet
-- getting log for operation: <tests.common.impala_connection.OperationHandle
object at 0x7f10199e1f90>
-- getting runtime profile operation:
<tests.common.impala_connection.OperationHandle object at 0x7f10199e1f90>
-- closing query for operation handle:
<tests.common.impala_connection.OperationHandle object at 0x7f10199e1f90>
-- 2024-08-05 07:13:57,673 INFO MainThread: Closing active operation
-- 2024-08-05 07:13:57,674 INFO MainThread: Loading query test file:
/data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/testdata/workloads/functional-query/queries/QueryTest/except.test
-- Executing select id, bool_col, tinyint_col, smallint_col, int_col,
bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col,
year, month from alltypestiny
{code}
was:
Recent changes seem to have resulted in memory oversubscription errors in
TestQuery tests.
h3. Error Message
{code}
query_test/test_queries.py:146: in test_except
self.run_test_case('QueryTest/except', vector) common/impala_test_suite.py:789:
in run_test_case user=test_section.get('USER', '').strip() or None)
common/impala_test_suite.py:708: in __exec_in_impala result =
self.__execute_query(target_impalad_client, query, user=user)
common/impala_test_suite.py:1079: in __execute_query return
impalad_client.execute(query, user=user) common/impala_connection.py:378: in
execute self.__cursor.execute(sql_stmt, configuration=self.__query_options)
/data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:343:
in execute self._wait_to_finish() # make execute synchronous
/data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:428:
in _wait_to_finish raise OperationalError(resp.errorMessage) E
OperationalError: Failed to get minimum memory reservation of 4.09 MB on daemon
impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000 for query
8e498fe323d4f770:0364fa4900000000 due to following error: Memory limit
exceeded: Could not allocate memory while trying to increase reservation. E
Query(8e498fe323d4f770:0364fa4900000000) could not allocate 4.09 MB without
exceeding limit. E Error occurred on backend
impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000 E Memory
left in process limit: 1.19 GB E Query(8e498fe323d4f770:0364fa4900000000):
Reservation=0 ReservationLimit=9.60 GB OtherMemory=0 Total=0 Peak=0 E Memory is
likely oversubscribed. Reducing query concurrency or configuring admission
control may help avoid this error.
{code}
h3. Stacktrace
{code}
query_test/test_queries.py:146: in test_except
self.run_test_case('QueryTest/except', vector) common/impala_test_suite.py:789:
in run_test_case user=test_section.get('USER', '').strip() or None)
common/impala_test_suite.py:708: in __exec_in_impala result =
self.__execute_query(target_impalad_client, query, user=user)
common/impala_test_suite.py:1079: in __execute_query return
impalad_client.execute(query, user=user) common/impala_connection.py:378: in
execute self.__cursor.execute(sql_stmt, configuration=self.__query_options)
/data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:343:
in execute self._wait_to_finish() # make execute synchronous
/data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:428:
in _wait_to_finish raise OperationalError(resp.errorMessage) E
OperationalError: Failed to get minimum memory reservation of 4.09 MB on daemon
impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000 for query
8e498fe323d4f770:0364fa4900000000 due to following error: Memory limit
exceeded: Could not allocate memory while trying to increase reservation. E
Query(8e498fe323d4f770:0364fa4900000000) could not allocate 4.09 MB without
exceeding limit. E Error occurred on backend
impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000 E Memory
left in process limit: 1.19 GB E Query(8e498fe323d4f770:0364fa4900000000):
Reservation=0 ReservationLimit=9.60 GB OtherMemory=0 Total=0 Peak=0 E Memory is
likely oversubscribed. Reducing query concurrency or configuring admission
control may help avoid this error.
{code}
h3. Standard Error
{code}
-- Executing use functional_parquet -- getting log for operation:
<tests.common.impala_connection.OperationHandle object at 0x7f10199e1f90> --
getting runtime profile operation:
<tests.common.impala_connection.OperationHandle object at 0x7f10199e1f90> --
closing query for operation handle:
<tests.common.impala_connection.OperationHandle object at 0x7f10199e1f90> --
2024-08-05 07:13:57,673 INFO MainThread: Closing active operation -- 2024-08-05
07:13:57,674 INFO MainThread: Loading query test file:
/data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/testdata/workloads/functional-query/queries/QueryTest/except.test
-- Executing select id, bool_col, tinyint_col, smallint_col, int_col,
bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col,
year, month from alltypestiny
{code}
> Cluster of memory oversubscription errors
> -----------------------------------------
>
> Key: IMPALA-13280
> URL: https://issues.apache.org/jira/browse/IMPALA-13280
> Project: IMPALA
> Issue Type: Bug
> Reporter: Andrew Sherman
> Priority: Critical
>
> Recent changes seem to have resulted in memory oversubscription errors in
> TestQuery tests.
> Error Message
> {code}
> query_test/test_queries.py:146: in test_except
> self.run_test_case('QueryTest/except', vector)
> common/impala_test_suite.py:789: in run_test_case
> user=test_section.get('USER', '').strip() or None)
> common/impala_test_suite.py:708: in __exec_in_impala result =
> self.__execute_query(target_impalad_client, query, user=user)
> common/impala_test_suite.py:1079: in __execute_query return
> impalad_client.execute(query, user=user) common/impala_connection.py:378: in
> execute self.__cursor.execute(sql_stmt,
> configuration=self.__query_options)
> /data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:343:
> in execute self._wait_to_finish() # make execute synchronous
> /data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:428:
> in _wait_to_finish raise OperationalError(resp.errorMessage) E
> OperationalError: Failed to get minimum memory reservation of 4.09 MB on
> daemon impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000 for
> query 8e498fe323d4f770:0364fa4900000000 due to following error: Memory limit
> exceeded: Could not allocate memory while trying to increase reservation. E
> Query(8e498fe323d4f770:0364fa4900000000) could not allocate 4.09 MB without
> exceeding limit. E Error occurred on backend
> impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000 E Memory
> left in process limit: 1.19 GB E Query(8e498fe323d4f770:0364fa4900000000):
> Reservation=0 ReservationLimit=9.60 GB OtherMemory=0 Total=0 Peak=0 E
> Memory is likely oversubscribed. Reducing query concurrency or configuring
> admission control may help avoid this error.
> {code}
> Stacktrace
> {code}
> query_test/test_queries.py:146: in test_except
> self.run_test_case('QueryTest/except', vector)
> common/impala_test_suite.py:789: in run_test_case
> user=test_section.get('USER', '').strip() or None)
> common/impala_test_suite.py:708: in __exec_in_impala
> result = self.__execute_query(target_impalad_client, query, user=user)
> common/impala_test_suite.py:1079: in __execute_query
> return impalad_client.execute(query, user=user)
> common/impala_connection.py:378: in execute
> self.__cursor.execute(sql_stmt, configuration=self.__query_options)
> /data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:343:
> in execute
> self._wait_to_finish() # make execute synchronous
> /data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/infra/python/env-gcc10.4.0/lib/python2.7/site-packages/impala/hiveserver2.py:428:
> in _wait_to_finish
> raise OperationalError(resp.errorMessage)
> E OperationalError: Failed to get minimum memory reservation of 4.09 MB on
> daemon impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000 for
> query 8e498fe323d4f770:0364fa4900000000 due to following error: Memory limit
> exceeded: Could not allocate memory while trying to increase reservation.
> E Query(8e498fe323d4f770:0364fa4900000000) could not allocate 4.09 MB
> without exceeding limit.
> E Error occurred on backend
> impala-ec2-centos79-m6i-4xlarge-xldisk-191f.vpc.cloudera.com:27000
> E Memory left in process limit: 1.19 GB
> E Query(8e498fe323d4f770:0364fa4900000000): Reservation=0
> ReservationLimit=9.60 GB OtherMemory=0 Total=0 Peak=0
> E Memory is likely oversubscribed. Reducing query concurrency or
> configuring admission control may help avoid this error.
> {code}
> Standard Error
> {code}
> -- Executing use functional_parquet
> -- getting log for operation: <tests.common.impala_connection.OperationHandle
> object at 0x7f10199e1f90>
> -- getting runtime profile operation:
> <tests.common.impala_connection.OperationHandle object at 0x7f10199e1f90>
> -- closing query for operation handle:
> <tests.common.impala_connection.OperationHandle object at 0x7f10199e1f90>
> -- 2024-08-05 07:13:57,673 INFO MainThread: Closing active operation
> -- 2024-08-05 07:13:57,674 INFO MainThread: Loading query test file:
> /data/jenkins/workspace/impala-cdw-master-exhaustive/repos/Impala/testdata/workloads/functional-query/queries/QueryTest/except.test
> -- Executing select id, bool_col, tinyint_col, smallint_col, int_col,
> bigint_col, float_col, double_col, date_string_col, string_col,
> timestamp_col, year, month from alltypestiny
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
