Repository: incubator-impala Updated Branches: refs/heads/master 9e438a08f -> 310edd5d0
IMPALA-3832: test invalid data handling in lzo text scanner This adds the lzo text scanner to the fuzz testing. I ran the test in a loop under ASAN overnight and didn't see any failures. Change-Id: Ib707014c1fcfb80cb8076f644fc2b62a5ae758d7 Reviewed-on: http://gerrit.cloudera.org:8080/4096 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/310edd5d Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/310edd5d Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/310edd5d Branch: refs/heads/master Commit: 310edd5d0040b2848f68354091d311fe9ee1c822 Parents: 9e438a0 Author: Tim Armstrong <[email protected]> Authored: Mon Aug 15 17:58:57 2016 -0700 Committer: Internal Jenkins <[email protected]> Committed: Wed Aug 24 03:09:32 2016 +0000 ---------------------------------------------------------------------- tests/query_test/test_scanners_fuzz.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/310edd5d/tests/query_test/test_scanners_fuzz.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_scanners_fuzz.py b/tests/query_test/test_scanners_fuzz.py index ae17572..76cc62a 100644 --- a/tests/query_test/test_scanners_fuzz.py +++ b/tests/query_test/test_scanners_fuzz.py @@ -39,10 +39,11 @@ class TestScannersFuzzing(ImpalaTestSuite): def add_test_dimensions(cls): super(TestScannersFuzzing, cls).add_test_dimensions() # TODO: enable for more table formats once they consistently pass the fuzz test. - cls.TestMatrix.add_constraint(lambda v:\ + cls.TestMatrix.add_constraint(lambda v: v.get_value('table_format').file_format in ('avro', 'parquet') or - (v.get_value('table_format').file_format == 'text' - and v.get_value('table_format').compression_type == 'none')) + (v.get_value('table_format').file_format == 'text' and + v.get_value('table_format').compression_codec in ('none', 'lzo'))) + def test_fuzz_alltypes(self, vector, unique_database): self.run_fuzz_test(vector, unique_database, "alltypes") @@ -50,11 +51,15 @@ class TestScannersFuzzing(ImpalaTestSuite): def test_fuzz_decimal_tbl(self, vector, unique_database): table_format = vector.get_value('table_format') table_name = "decimal_tbl" - if table_format.file_format in ('avro'): + if table_format.file_format == 'avro': table_name = "avro_decimal_tbl" - if table_format.compression_codec != 'block' or \ - table_format.compression_type != 'snap': + if table_format.compression_codec != 'snap' or \ + table_format.compression_type != 'block': pytest.skip() + elif table_format.file_format == 'text' and \ + table_format.compression_codec != 'none': + # decimal_tbl is not present for these file formats + pytest.skip() self.run_fuzz_test(vector, unique_database, table_name, 10) @@ -134,8 +139,12 @@ class TestScannersFuzzing(ImpalaTestSuite): continue msg = "Should not throw error when abort_on_error=0: '{0}'".format(e) LOG.error(msg) - # Parquet fails the query for some parse errors. - if table_format.file_format == 'parquet': + # Parquet and compressed text can fail the query for some parse errors. + # E.g. corrupt Parquet footer (IMPALA-3773) or a corrupt LZO index file + # (IMPALA-4013). + if table_format.file_format == 'parquet' or \ + (table_format.file_format == 'text' and + table_format.compression_codec != 'none'): xfail_msgs.append(msg) else: raise
