This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 11d2fe4fc IMPALA-12788: Fix HBaseTable still get loaded even if HBase
is down
11d2fe4fc is described below
commit 11d2fe4fc00a1e6ef2d3a45825be9845456adc1d
Author: stiga-huang <[email protected]>
AuthorDate: Tue Feb 6 06:05:53 2024 +0800
IMPALA-12788: Fix HBaseTable still get loaded even if HBase is down
When loading a table backed by HBase, it's intended to check whether the
table exists in HBase in HBaseTable.load() and loadFromThrift().
However, the current check just gets the table object and then closes
it. This won't fail even if HBase is down. See JIRA description for the
stacktrace.
This patch fixes the check to fetch the column family names which is a
light-weight request and will fail if HBase is down or the table doesn't
exist in HBase.
Splits the following tests to skip the HBase part when running on S3:
- TestNestedStructsInSelectList.test_struct_in_select_list
- TestDdlStatements.test_alter_set_column_stats
- TestShowCreateTable.test_show_create_table
Tests:
- Run CORE tests on S3
Change-Id: Ib497f11ecc338d0f84d3d7bd8ccfcf8da4def0cb
Reviewed-on: http://gerrit.cloudera.org:8080/21003
Reviewed-by: Quanlong Huang <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
.../java/org/apache/impala/catalog/HBaseTable.java | 8 +--
.../alter-hbase-table-set-column-stats.test | 59 ++++++++++++++++++++++
.../QueryTest/alter-table-set-column-stats.test | 59 ----------------------
.../queries/QueryTest/show-create-table-hbase.test | 25 +++++++++
.../queries/QueryTest/show-create-table.test | 24 ---------
.../QueryTest/struct-in-select-list-hbase.test | 8 +++
.../queries/QueryTest/struct-in-select-list.test | 7 ---
tests/metadata/test_ddl.py | 6 +++
tests/metadata/test_show_create_table.py | 7 ++-
tests/query_test/test_nested_types.py | 5 ++
10 files changed, 113 insertions(+), 95 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
b/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
index c4f293960..4248f0395 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
@@ -109,9 +109,9 @@ public class HBaseTable extends Table implements
FeHBaseTable {
List<Column> cols;
try {
hbaseTableName_ = Util.getHBaseTableName(getMetaStoreTable());
- // Warm up the connection and verify the table exists.
- Util.getHBaseTable(hbaseTableName_).close();
columnFamilies_ = null;
+ // Warm up the connection and verify the table exists.
+ getColumnFamilies();
cols = Util.loadColumns(msTable_);
} finally {
storageMetadataLoadTime_ = storageLoadTimer.stop();
@@ -138,9 +138,9 @@ public class HBaseTable extends Table implements
FeHBaseTable {
super.loadFromThrift(table);
try {
hbaseTableName_ = Util.getHBaseTableName(getMetaStoreTable());
- // Warm up the connection and verify the table exists.
- Util.getHBaseTable(hbaseTableName_).close();
columnFamilies_ = null;
+ // Warm up the connection and verify the table exists.
+ getColumnFamilies();
} catch (Exception e) {
throw new TableLoadingException(
"Failed to load metadata for HBase table from thrift table: " +
name_, e);
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/alter-hbase-table-set-column-stats.test
b/testdata/workloads/functional-query/queries/QueryTest/alter-hbase-table-set-column-stats.test
new file mode 100644
index 000000000..b8ceb6786
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/alter-hbase-table-set-column-stats.test
@@ -0,0 +1,59 @@
+====
+---- QUERY
+create external table alltypes_hbase_clone like functional_hbase.alltypes
+====
+---- QUERY
+alter table alltypes_hbase_clone set column stats double_col ('numDVs'='2');
+alter table alltypes_hbase_clone set column stats timestamp_col
('numNulls'='9');
+alter table alltypes_hbase_clone set column stats int_col
('numDVs'='100','numNulls'='20');
+alter table alltypes_hbase_clone set column stats string_col
('maxSize'='555','avgSize'='60');
+====
+---- QUERY
+show column stats alltypes_hbase_clone
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE, #TRUES, #FALSES
+---- RESULTS
+'id','INT',-1,-1,4,4,-1,-1
+'bool_col','BOOLEAN',-1,-1,1,1,-1,-1
+'tinyint_col','TINYINT',-1,-1,1,1,-1,-1
+'smallint_col','SMALLINT',-1,-1,2,2,-1,-1
+'int_col','INT',100,20,4,4,-1,-1
+'bigint_col','BIGINT',-1,-1,8,8,-1,-1
+'float_col','FLOAT',-1,-1,4,4,-1,-1
+'double_col','DOUBLE',2,-1,8,8,-1,-1
+'date_string_col','STRING',-1,-1,-1,-1,-1,-1
+'string_col','STRING',-1,-1,555,60,-1,-1
+'timestamp_col','TIMESTAMP',-1,9,16,16,-1,-1
+'year','INT',-1,-1,4,4,-1,-1
+'month','INT',-1,-1,4,4,-1,-1
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
+====
+---- QUERY
+# Reset the column stats to an unknown state by setting the values to -1
+alter table alltypes_hbase_clone set column stats double_col ('numDVs'='-1');
+alter table alltypes_hbase_clone set column stats timestamp_col
('numNulls'='-1');
+alter table alltypes_hbase_clone set column stats int_col
('numDVs'='-1','numNulls'='-1');
+alter table alltypes_hbase_clone set column stats string_col
('maxSize'='-1','avgSize'='-1');
+====
+---- QUERY
+show column stats alltypes_hbase_clone
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE, #TRUES, #FALSES
+---- RESULTS
+'id','INT',-1,-1,4,4,-1,-1
+'bool_col','BOOLEAN',-1,-1,1,1,-1,-1
+'tinyint_col','TINYINT',-1,-1,1,1,-1,-1
+'smallint_col','SMALLINT',-1,-1,2,2,-1,-1
+'int_col','INT',-1,-1,4,4,-1,-1
+'bigint_col','BIGINT',-1,-1,8,8,-1,-1
+'float_col','FLOAT',-1,-1,4,4,-1,-1
+'double_col','DOUBLE',-1,-1,8,8,-1,-1
+'date_string_col','STRING',-1,-1,-1,-1,-1,-1
+'string_col','STRING',-1,-1,-1,-1,-1,-1
+'timestamp_col','TIMESTAMP',-1,-1,16,16,-1,-1
+'year','INT',-1,-1,4,4,-1,-1
+'month','INT',-1,-1,4,4,-1,-1
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/alter-table-set-column-stats.test
b/testdata/workloads/functional-query/queries/QueryTest/alter-table-set-column-stats.test
index 2987267d3..3c260a34f 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/alter-table-set-column-stats.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/alter-table-set-column-stats.test
@@ -90,62 +90,3 @@ where year = 2009 and month between 2 and 3 and int_col = 9
and id between 300 a
---- TYPES
INT, INT, DOUBLE, STRING, TIMESTAMP
====
----- QUERY
-# Similar test on an HBase table.
-create external table alltypes_hbase_clone like functional_hbase.alltypes
-====
----- QUERY
-alter table alltypes_hbase_clone set column stats double_col ('numDVs'='2');
-alter table alltypes_hbase_clone set column stats timestamp_col
('numNulls'='9');
-alter table alltypes_hbase_clone set column stats int_col
('numDVs'='100','numNulls'='20');
-alter table alltypes_hbase_clone set column stats string_col
('maxSize'='555','avgSize'='60');
-====
----- QUERY
-show column stats alltypes_hbase_clone
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE, #TRUES, #FALSES
----- RESULTS
-'id','INT',-1,-1,4,4,-1,-1
-'bool_col','BOOLEAN',-1,-1,1,1,-1,-1
-'tinyint_col','TINYINT',-1,-1,1,1,-1,-1
-'smallint_col','SMALLINT',-1,-1,2,2,-1,-1
-'int_col','INT',100,20,4,4,-1,-1
-'bigint_col','BIGINT',-1,-1,8,8,-1,-1
-'float_col','FLOAT',-1,-1,4,4,-1,-1
-'double_col','DOUBLE',2,-1,8,8,-1,-1
-'date_string_col','STRING',-1,-1,-1,-1,-1,-1
-'string_col','STRING',-1,-1,555,60,-1,-1
-'timestamp_col','TIMESTAMP',-1,9,16,16,-1,-1
-'year','INT',-1,-1,4,4,-1,-1
-'month','INT',-1,-1,4,4,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
----- QUERY
-# Reset the column stats to an unknown state by setting the values to -1
-alter table alltypes_hbase_clone set column stats double_col ('numDVs'='-1');
-alter table alltypes_hbase_clone set column stats timestamp_col
('numNulls'='-1');
-alter table alltypes_hbase_clone set column stats int_col
('numDVs'='-1','numNulls'='-1');
-alter table alltypes_hbase_clone set column stats string_col
('maxSize'='-1','avgSize'='-1');
-====
----- QUERY
-show column stats alltypes_hbase_clone
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE, #TRUES, #FALSES
----- RESULTS
-'id','INT',-1,-1,4,4,-1,-1
-'bool_col','BOOLEAN',-1,-1,1,1,-1,-1
-'tinyint_col','TINYINT',-1,-1,1,1,-1,-1
-'smallint_col','SMALLINT',-1,-1,2,2,-1,-1
-'int_col','INT',-1,-1,4,4,-1,-1
-'bigint_col','BIGINT',-1,-1,8,8,-1,-1
-'float_col','FLOAT',-1,-1,4,4,-1,-1
-'double_col','DOUBLE',-1,-1,8,8,-1,-1
-'date_string_col','STRING',-1,-1,-1,-1,-1,-1
-'string_col','STRING',-1,-1,-1,-1,-1,-1
-'timestamp_col','TIMESTAMP',-1,-1,16,16,-1,-1
-'year','INT',-1,-1,4,4,-1,-1
-'month','INT',-1,-1,4,4,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/show-create-table-hbase.test
b/testdata/workloads/functional-query/queries/QueryTest/show-create-table-hbase.test
new file mode 100644
index 000000000..6ddb49d13
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/show-create-table-hbase.test
@@ -0,0 +1,25 @@
+====
+---- QUERY
+SHOW CREATE TABLE functional_hbase.alltypes
+---- RESULTS-HIVE
+CREATE EXTERNAL TABLE functional_hbase.alltypes (
+ id INT COMMENT 'Add a comment',
+ bigint_col BIGINT,
+ bool_col BOOLEAN,
+ date_string_col STRING,
+ double_col DOUBLE,
+ float_col FLOAT,
+ int_col INT,
+ month INT,
+ smallint_col SMALLINT,
+ string_col STRING,
+ timestamp_col TIMESTAMP,
+ tinyint_col TINYINT,
+ year INT
+)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES
('hbase.columns.mapping'=':key,d:bool_col,d:tinyint_col,d:smallint_col,d:int_col,d:bigint_col,d:float_col,d:double_col,d:date_string_col,d:string_col,d:timestamp_col,d:year,d:month',
+ 'serialization.format'='1')
+TBLPROPERTIES ('hbase.table.name'='functional_hbase.alltypes',
+
'storage_handler'='org.apache.hadoop.hive.hbase.HBaseStorageHandler')
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
index 018785666..af726cf29 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
@@ -558,30 +558,6 @@ STORED AS TEXTFILE
LOCATION '$$location_uri$$'
====
---- QUERY
-SHOW CREATE TABLE functional_hbase.alltypes
----- RESULTS-HIVE
-CREATE EXTERNAL TABLE functional_hbase.alltypes (
- id INT COMMENT 'Add a comment',
- bigint_col BIGINT,
- bool_col BOOLEAN,
- date_string_col STRING,
- double_col DOUBLE,
- float_col FLOAT,
- int_col INT,
- month INT,
- smallint_col SMALLINT,
- string_col STRING,
- timestamp_col TIMESTAMP,
- tinyint_col TINYINT,
- year INT
-)
-STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
-WITH SERDEPROPERTIES
('hbase.columns.mapping'=':key,d:bool_col,d:tinyint_col,d:smallint_col,d:int_col,d:bigint_col,d:float_col,d:double_col,d:date_string_col,d:string_col,d:timestamp_col,d:year,d:month',
- 'serialization.format'='1')
-TBLPROPERTIES ('hbase.table.name'='functional_hbase.alltypes',
-
'storage_handler'='org.apache.hadoop.hive.hbase.HBaseStorageHandler')
-====
----- QUERY
SHOW CREATE TABLE functional.parent_table
---- RESULTS-HIVE
CREATE EXTERNAL TABLE functional.parent_table (
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list-hbase.test
b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list-hbase.test
new file mode 100644
index 000000000..f957f0ef0
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list-hbase.test
@@ -0,0 +1,8 @@
+====
+---- QUERY
+# It's not supported to create a view with structs from a table type that
doesn't
+# support selecting structs.
+create view tmp_view as select id, int_struct_col from
functional_hbase.allcomplextypes;
+---- CATCH
+is not supported when querying STRUCT type STRUCT<f1:INT,f2:INT>
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
index 546d1c527..f7ebf8ce7 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
@@ -565,13 +565,6 @@ where struct_val.tinyint_col=8 and
strleft(struct_val.date_string_col, 5) = "12/
INT,STRING
====
---- QUERY
-# It's not supported to create a view with structs from a table type that
doesn't
-# support selecting structs.
-create view tmp_view as select id, int_struct_col from
functional_hbase.allcomplextypes;
----- CATCH
-is not supported when querying STRUCT type STRUCT<f1:INT,f2:INT>
-====
----- QUERY
# It's not supported to create a view with structs from a file format that
doesn't
# support selecting structs.
create view tmp_view as select id, int_struct_col from
functional.allcomplextypes;
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 394afd0b9..bb5b346c4 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -472,6 +472,12 @@ class TestDdlStatements(TestDdlBase):
self.run_test_case('QueryTest/alter-table-set-column-stats', vector,
use_db=unique_database,
multiple_impalad=self._use_multiple_impalad(vector))
+ @SkipIfFS.hbase
+ @UniqueDatabase.parametrize(sync_ddl=True)
+ def test_alter_hbase_set_column_stats(self, vector, unique_database):
+ self.run_test_case('QueryTest/alter-hbase-table-set-column-stats', vector,
+ use_db=unique_database,
multiple_impalad=self._use_multiple_impalad(vector))
+
@SkipIfLocal.hdfs_client
def test_drop_partition_with_purge(self, vector, unique_database):
"""Verfies whether alter <tbl> drop partition purge actually skips trash"""
diff --git a/tests/metadata/test_show_create_table.py
b/tests/metadata/test_show_create_table.py
index de95d2b7f..0bab6038b 100644
--- a/tests/metadata/test_show_create_table.py
+++ b/tests/metadata/test_show_create_table.py
@@ -21,7 +21,7 @@ import re
import shlex
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIf, SkipIfHive2
+from tests.common.skip import SkipIfFS, SkipIfHive2
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.test_file_parser import QueryTestSectionReader, remove_comments
from tests.common.environ import HIVE_MAJOR_VERSION
@@ -81,6 +81,11 @@ class TestShowCreateTable(ImpalaTestSuite):
self.__run_show_create_table_test_case('QueryTest/show-create-table',
vector,
unique_database)
+ @SkipIfFS.hbase
+ def test_show_create_table_hbase(self, vector, unique_database):
+
self.__run_show_create_table_test_case('QueryTest/show-create-table-hbase',
vector,
+ unique_database)
+
@SkipIfHive2.acid
def test_show_create_table_full_acid(self, vector, unique_database):
self.__run_show_create_table_test_case('QueryTest/show-create-table-full-acid',
diff --git a/tests/query_test/test_nested_types.py
b/tests/query_test/test_nested_types.py
index 75b3171b9..12179701e 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -146,6 +146,11 @@ class TestNestedStructsInSelectList(ImpalaTestSuite):
new_vector.get_value('exec_option')['TIMEZONE'] = '"Europe/Budapest"'
self.run_test_case('QueryTest/struct-in-select-list', new_vector)
+ @SkipIfFS.hbase
+ def test_struct_in_select_list_hbase(self, vector):
+ """Verify error cases that are not supported on HBase tables"""
+ self.run_test_case('QueryTest/struct-in-select-list-hbase', vector)
+
def test_nested_struct_in_select_list(self, vector):
"""Queries where a nested struct column is in the select list"""
new_vector = deepcopy(vector)
