[ 
https://issues.apache.org/jira/browse/IMPALA-5772?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Tim Armstrong resolved IMPALA-5772.
-----------------------------------
       Resolution: Fixed
    Fix Version/s: Impala 2.10.0

> Expected failure in test_scratch_disk.TestScratchDir didn't occur
> -----------------------------------------------------------------
>
>                 Key: IMPALA-5772
>                 URL: https://issues.apache.org/jira/browse/IMPALA-5772
>             Project: IMPALA
>          Issue Type: Bug
>          Components: Backend
>    Affects Versions: Impala 2.10.0
>            Reporter: Michael Ho
>            Assignee: Tim Armstrong
>            Priority: Blocker
>              Labels: broken-build
>             Fix For: Impala 2.10.0
>
>
> Expected failure in test_non_writable_dirs and test_non_existing_dirs didn't 
> occur on S3 builds. Not sure if there were some changes to S3 related code 
> recently.
> cc'ing [~sailesh] and [~tarmstrong]
> {noformat}
> 09:04:19 =================================== FAILURES 
> ===================================
> 09:04:19  TestScratchDir.test_no_dirs[exec_option: {'batch_size': 0, 
> 'num_nodes': 0, 'disable_codegen_rows_threshold': 0, 'disable_codegen': 
> False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | 
> table_format: text/none] 
> 09:04:19 
> 09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x36dfad0>
> 09:04:19 vector = <tests.common.test_vector.ImpalaTestVector object at 
> 0x56b4950>
> 09:04:19 
> 09:04:19     @pytest.mark.execute_serially
> 09:04:19     def test_no_dirs(self, vector):
> 09:04:19       """ Test we can execute a query with no scratch dirs """
> 09:04:19       self._start_impala_cluster(['--impalad_args="-logbuflevel=-1 
> -scratch_dirs="'])
> 09:04:19       self.assert_impalad_log_contains("WARNING",
> 09:04:19           "Running without spill to disk: no scratch directories 
> provided\.")
> 09:04:19       exec_option = vector.get_value('exec_option')
> 09:04:19       exec_option['buffer_pool_limit'] = self.buffer_pool_limit
> 09:04:19       impalad = self.cluster.get_any_impalad()
> 09:04:19       client = impalad.service.create_beeswax_client()
> 09:04:19       # Expect spill to disk to fail
> 09:04:19 >     self.execute_query_expect_failure(client, self.spill_query, 
> exec_option)
> 09:04:19 
> 09:04:19 custom_cluster/test_scratch_disk.py:107: 
> 09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
> _ _ _ _ _ _ 
> 09:04:19 common/impala_test_suite.py:506: in wrapper
> 09:04:19     return function(*args, **kwargs)
> 09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
> _ _ _ _ _ _ 
> 09:04:19 
> 09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x36dfad0>
> 09:04:19 impalad_client = <tests.common.impala_connection.BeeswaxConnection 
> object at 0x36dfb90>
> 09:04:19 query = '\n      select o_orderdate, o_custkey, o_comment\n      
> from tpch.orders\n      order by o_orderdate\n      '
> 09:04:19 query_options = {'abort_on_error': 1, 'batch_size': 0, 
> 'buffer_pool_limit': '64m', 'disable_codegen': False, ...}
> 09:04:19 
> 09:04:19     @execute_wrapper
> 09:04:19     def execute_query_expect_failure(self, impalad_client, query, 
> query_options=None):
> 09:04:19       """Executes a query and asserts if the query succeeds"""
> 09:04:19       result = None
> 09:04:19       try:
> 09:04:19         result = self.__execute_query(impalad_client, query, 
> query_options)
> 09:04:19       except Exception, e:
> 09:04:19         return e
> 09:04:19     
> 09:04:19 >     assert not result.success, "No failure encountered for query 
> %s" % query
> 09:04:19 E     AssertionError: No failure encountered for query 
> 09:04:19 E           select o_orderdate, o_custkey, o_comment
> 09:04:19 E           from tpch.orders
> 09:04:19 E           order by o_orderdate
> 09:04:19 
> 09:04:19 common/impala_test_suite.py:526: AssertionError
> 09:04:19 ----------------------------- Captured stdout call 
> -----------------------------
> {noformat}
> {noformat}
> 09:04:19 -- connecting to: 
> impala-boost-static-burst-slave-1780.vpc.cloudera.com:21002
> 09:04:19 SET batch_size=0;
> 09:04:19 SET num_nodes=0;
> 09:04:19 SET disable_codegen_rows_threshold=0;
> 09:04:19 SET disable_codegen=False;
> 09:04:19 SET abort_on_error=1;
> 09:04:19 SET exec_single_node_rows_threshold=0;
> 09:04:19 SET buffer_pool_limit=64m;
> 09:04:19 -- executing against 
> impala-boost-static-burst-slave-1780.vpc.cloudera.com:21002
> 09:04:19 
> 09:04:19       select o_orderdate, o_custkey, o_comment
> 09:04:19       from tpch.orders
> 09:04:19       order by o_orderdate
> 09:04:19       ;
> 09:04:19 
> 09:04:19  TestScratchDir.test_non_writable_dirs[exec_option: {'batch_size': 
> 0, 'num_nodes': 0, 'disable_codegen_rows_threshold': 0, 'disable_codegen': 
> False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | 
> table_format: text/none] 
> 09:04:19 
> 09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x4d1b450>
> 09:04:19 vector = <tests.common.test_vector.ImpalaTestVector object at 
> 0x56b49d0>
> 09:04:19 
> 09:04:19     @pytest.mark.execute_serially
> 09:04:19     def test_non_writable_dirs(self, vector):
> 09:04:19       """ Test we can execute a query with only bad non-writable 
> scratch """
> 09:04:19       non_writable_dirs = self.generate_dirs(5, writable=False)
> 09:04:19       self._start_impala_cluster([
> 09:04:19         '--impalad_args="-logbuflevel=-1 -scratch_dirs={0}"'.format(
> 09:04:19         ','.join(non_writable_dirs))])
> 09:04:19       self.assert_impalad_log_contains("ERROR", "Running without 
> spill to disk: could "
> 09:04:19           + "not use any scratch directories in list:.*. See 
> previous "
> 09:04:19           + "warnings for information on causes.")
> 09:04:19       self.assert_impalad_log_contains("WARNING", "Could not remove 
> and recreate directory "
> 09:04:19               + ".*: cannot use it for scratch\. Error was: .*", 
> expected_count=5)
> 09:04:19       exec_option = vector.get_value('exec_option')
> 09:04:19       exec_option['buffer_pool_limit'] = self.buffer_pool_limit
> 09:04:19       impalad = self.cluster.get_any_impalad()
> 09:04:19       client = impalad.service.create_beeswax_client()
> 09:04:19       # Expect spill to disk to fail
> 09:04:19 >     self.execute_query_expect_failure(client, self.spill_query, 
> exec_option)
> 09:04:19 
> 09:04:19 custom_cluster/test_scratch_disk.py:128: 
> 09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
> _ _ _ _ _ _ 
> 09:04:19 common/impala_test_suite.py:506: in wrapper
> 09:04:19     return function(*args, **kwargs)
> 09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
> _ _ _ _ _ _ 
> 09:04:19 
> 09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x4d1b450>
> 09:04:19 impalad_client = <tests.common.impala_connection.BeeswaxConnection 
> object at 0x4d1bed0>
> 09:04:19 query = '\n      select o_orderdate, o_custkey, o_comment\n      
> from tpch.orders\n      order by o_orderdate\n      '
> 09:04:19 query_options = {'abort_on_error': 1, 'batch_size': 0, 
> 'buffer_pool_limit': '64m', 'disable_codegen': False, ...}
> 09:04:19 
> 09:04:19     @execute_wrapper
> 09:04:19     def execute_query_expect_failure(self, impalad_client, query, 
> query_options=None):
> 09:04:19       """Executes a query and asserts if the query succeeds"""
> 09:04:19       result = None
> 09:04:19       try:
> 09:04:19         result = self.__execute_query(impalad_client, query, 
> query_options)
> 09:04:19       except Exception, e:
> 09:04:19         return e
> 09:04:19     
> 09:04:19 >     assert not result.success, "No failure encountered for query 
> %s" % query
> 09:04:19 E     AssertionError: No failure encountered for query 
> 09:04:19 E           select o_orderdate, o_custkey, o_comment
> 09:04:19 E           from tpch.orders
> 09:04:19 E           order by o_orderdate
> 09:04:19 
> 09:04:19 common/impala_test_suite.py:526: AssertionError
> 09:04:19 ----------------------------- Captured stdout call 
> -----------------------------
> {noformat}
> {noformat}
> 09:04:19 -- connecting to: 
> impala-boost-static-burst-slave-1780.vpc.cloudera.com:21000
> 09:04:19 SET batch_size=0;
> 09:04:19 SET num_nodes=0;
> 09:04:19 SET disable_codegen_rows_threshold=0;
> 09:04:19 SET disable_codegen=False;
> 09:04:19 SET abort_on_error=1;
> 09:04:19 SET exec_single_node_rows_threshold=0;
> 09:04:19 SET buffer_pool_limit=64m;
> 09:04:19 -- executing against 
> impala-boost-static-burst-slave-1780.vpc.cloudera.com:21000
> 09:04:19 
> 09:04:19       select o_orderdate, o_custkey, o_comment
> 09:04:19       from tpch.orders
> 09:04:19       order by o_orderdate
> 09:04:19       ;
> 09:04:19 
> 09:04:19  TestScratchDir.test_non_existing_dirs[exec_option: {'batch_size': 
> 0, 'num_nodes': 0, 'disable_codegen_rows_threshold': 0, 'disable_codegen': 
> False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | 
> table_format: text/none] 
> 09:04:19 
> 09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x4d2bad0>
> 09:04:19 vector = <tests.common.test_vector.ImpalaTestVector object at 
> 0x56b4410>
> 09:04:19 
> 09:04:19     @pytest.mark.execute_serially
> 09:04:19     def test_non_existing_dirs(self, vector):
> 09:04:19       """ Test that non-existing directories are not created or used 
> """
> 09:04:19       non_existing_dirs = self.generate_dirs(5, non_existing=True)
> 09:04:19       self._start_impala_cluster([
> 09:04:19         '--impalad_args="-logbuflevel=-1 -scratch_dirs={0}"'.format(
> 09:04:19         ','.join(non_existing_dirs))])
> 09:04:19       self.assert_impalad_log_contains("ERROR", "Running without 
> spill to disk: could "
> 09:04:19           + "not use any scratch directories in list:.*. See 
> previous "
> 09:04:19           + "warnings for information on causes.")
> 09:04:19       self.assert_impalad_log_contains("WARNING", "Cannot use 
> directory .* for scratch: "
> 09:04:19           + "Encountered exception while verifying existence of 
> directory path",
> 09:04:19           expected_count=5)
> 09:04:19       exec_option = vector.get_value('exec_option')
> 09:04:19       exec_option['buffer_pool_limit'] = self.buffer_pool_limit
> 09:04:19       impalad = self.cluster.get_any_impalad()
> 09:04:19       client = impalad.service.create_beeswax_client()
> 09:04:19       # Expect spill to disk to fail
> 09:04:19 >     self.execute_query_expect_failure(client, self.spill_query, 
> exec_option)
> 09:04:19 
> 09:04:19 custom_cluster/test_scratch_disk.py:151: 
> 09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
> _ _ _ _ _ _ 
> 09:04:19 common/impala_test_suite.py:506: in wrapper
> 09:04:19     return function(*args, **kwargs)
> 09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
> _ _ _ _ _ _ 
> 09:04:19 
> 09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x4d2bad0>
> 09:04:19 impalad_client = <tests.common.impala_connection.BeeswaxConnection 
> object at 0x4d2b290>
> 09:04:19 query = '\n      select o_orderdate, o_custkey, o_comment\n      
> from tpch.orders\n      order by o_orderdate\n      '
> 09:04:19 query_options = {'abort_on_error': 1, 'batch_size': 0, 
> 'buffer_pool_limit': '64m', 'disable_codegen': False, ...}
> 09:04:19 
> 09:04:19     @execute_wrapper
> 09:04:19     def execute_query_expect_failure(self, impalad_client, query, 
> query_options=None):
> 09:04:19       """Executes a query and asserts if the query succeeds"""
> 09:04:19       result = None
> 09:04:19       try:
> 09:04:19         result = self.__execute_query(impalad_client, query, 
> query_options)
> 09:04:19       except Exception, e:
> 09:04:19         return e
> 09:04:19     
> 09:04:19 >     assert not result.success, "No failure encountered for query 
> %s" % query
> 09:04:19 E     AssertionError: No failure encountered for query 
> 09:04:19 E           select o_orderdate, o_custkey, o_comment
> 09:04:19 E           from tpch.orders
> 09:04:19 E           order by o_orderdate
> 09:04:19 
> 09:04:19 common/impala_test_suite.py:526: AssertionError
> 09:04:19 ----------------------------- Captured stdout call 
> -----------------------------
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Reply via email to