This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit afb4e0b79645c5afe46aea4ba94eed134d063e3f Author: Sai Hemanth Gantasala <[email protected]> AuthorDate: Thu Oct 13 16:19:25 2022 -0700 IMPALA-11661: Added new api in MetastoreServiceHandler for find_next_compact2 method In the MetastoreServiceHandler class, the 'find_next_compact2' API implementation is missing which is causing test failures. This needs to be fixed by adding implementation of the 'find_next_compact2' api in the MetastoreServiceHandler class Testing: Added a test 'test_compaction_apis()' to verify that the 'find_next_compact2' api in HMS is reachable from impala. The same test can be used in the future to test newly added apis in HMS. Change-Id: I9f1663c16d2649c9c455e6dffde02894819b2761 Reviewed-on: http://gerrit.cloudera.org:8080/19140 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../catalog/metastore/MetastoreServiceHandler.java | 9 +++++ tests/custom_cluster/test_metastore_service.py | 38 ++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/fe/src/main/java/org/apache/impala/catalog/metastore/MetastoreServiceHandler.java b/fe/src/main/java/org/apache/impala/catalog/metastore/MetastoreServiceHandler.java index 2566703c3..c7cfb7635 100644 --- a/fe/src/main/java/org/apache/impala/catalog/metastore/MetastoreServiceHandler.java +++ b/fe/src/main/java/org/apache/impala/catalog/metastore/MetastoreServiceHandler.java @@ -85,6 +85,7 @@ import org.apache.hadoop.hive.metastore.api.DropPartitionsResult; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.ExtendedTableInfo; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.FindNextCompactRequest; import org.apache.hadoop.hive.metastore.api.FindSchemasByColsResp; import org.apache.hadoop.hive.metastore.api.FindSchemasByColsRqst; import org.apache.hadoop.hive.metastore.api.FireEventRequest; @@ -2399,6 +2400,14 @@ public abstract class MetastoreServiceHandler extends AbstractThriftHiveMetastor } } + @Override + public OptionalCompactionInfoStruct find_next_compact2(FindNextCompactRequest rqst) + throws MetaException, TException { + try (MetaStoreClient client = catalog_.getMetaStoreClient()) { + return client.getHiveClient().getThriftClient().find_next_compact2(rqst); + } + } + @Override public void update_compactor_state(CompactionInfoStruct compactionInfoStruct, long l) throws TException { diff --git a/tests/custom_cluster/test_metastore_service.py b/tests/custom_cluster/test_metastore_service.py index e7c18cfb3..05c7a0895 100644 --- a/tests/custom_cluster/test_metastore_service.py +++ b/tests/custom_cluster/test_metastore_service.py @@ -18,6 +18,7 @@ import pytest from hive_metastore.ttypes import Database from hive_metastore.ttypes import FieldSchema +from hive_metastore.ttypes import FindNextCompactRequest from hive_metastore.ttypes import GetTableRequest from hive_metastore.ttypes import GetPartitionsByNamesRequest from hive_metastore.ttypes import TruncateTableRequest @@ -973,6 +974,43 @@ class TestMetastoreService(CustomClusterTestSuite): if self.__get_database_no_throw(db_name) is not None: self.hive_client.drop_database(db_name, True, True) + @pytest.mark.execute_serially + @CustomClusterTestSuite.with_args( + impalad_args="--use_local_catalog=true", + catalogd_args="--catalog_topic_mode=minimal " + "--start_hms_server=true " + "--hms_port=5899 " + "--fallback_to_hms_on_errors=true " + ) + def test_compaction_apis(self): + """ + The test verifies if the following HMS compaction apis are + reachable from impala (so that MetastoreServiceHandler in impala + can talk to HMS through these APIs): + 1. find_next_compact2 + """ + catalog_hms_client = None + tbl_name = ImpalaTestSuite.get_random_name( + "test_table_compaction_seq_tbl_") + try: + catalog_hms_client, hive_transport = \ + ImpalaTestSuite.create_hive_client(5899) + assert catalog_hms_client is not None + # create managed table + self.run_stmt_in_hive("create transactional table default.{0} (c1 int)" + .format(tbl_name)) + self.run_stmt_in_hive("alter table default.{0} compact 'minor'" + .format(tbl_name)) + compactRequest = FindNextCompactRequest() + compactRequest.workerId = "myworker" + compactRequest.workerVersion = "4.0.0" + optionalCi = catalog_hms_client.find_next_compact2(compactRequest) + # If the above call is successful then find_next_compact2 api is reachable. + catalog_hms_client.drop_table("default", tbl_name, True) + finally: + if catalog_hms_client is not None: + catalog_hms_client.shutdown() + def __create_test_tbls_from_hive(self, db_name): """Util method to create test tables from hive in the given database. It creates 4 tables (partitioned and unpartitioned) for non-acid and acid cases and returns
