This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit ff503e4a4b4f5fcace6fea8659a881ccff075b5c Author: stiga-huang <[email protected]> AuthorDate: Sun Apr 23 17:22:04 2023 +0800 IMPALA-12084: IM under unloaded db should not send full db updates in catalog-v2 INVALIDATE METADATA under unloaded db will also add the newly added db along with the newly added table in the catalog update. In LocalCatalog mode, we should only send back invalidation of the db instead of the full db object. This is a missing logic we should have added in IMPALA-9936. Tests: - Add e2e test on LocalCatalog mode Change-Id: I38a44f69cca20331ab31acc270c2921a7bfedae5 Reviewed-on: http://gerrit.cloudera.org:8080/19787 Reviewed-by: Daniel Becker <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Michael Smith <[email protected]> --- .../org/apache/impala/service/CatalogOpExecutor.java | 5 ++--- .../custom_cluster/test_metadata_no_events_processing.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java index 56f08e28c..73bd44474 100644 --- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java +++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java @@ -6356,7 +6356,7 @@ public class CatalogOpExecutor { /** * Executes a TResetMetadataRequest and returns the result as a * TResetMetadataResponse. Based on the request parameters, this operation - * may do one of three things: + * may do one of these things: * 1) invalidate the entire catalog, forcing the metadata for all catalog * objects to be reloaded. * 2) invalidate a specific table, forcing the metadata to be reloaded @@ -6496,7 +6496,7 @@ public class CatalogOpExecutor { updatedThriftTable.getTable().getDb_name() + " was removed by a " + "concurrent operation. Try invalidating the table again."); } - resp.getResult().addToUpdated_catalog_objects(addedDb.toTCatalogObject()); + addDbToCatalogUpdate(addedDb, req.header.want_minimal_response, resp.getResult()); } resp.getResult().setVersion(updatedThriftTable.getCatalog_version()); } else if (req.isAuthorization()) { @@ -7214,7 +7214,6 @@ public class CatalogOpExecutor { Preconditions.checkNotNull(db); TCatalogObject updatedCatalogObject = wantMinimalResult ? db.toMinimalTCatalogObject() : db.toTCatalogObject(); - updatedCatalogObject.setCatalog_version(updatedCatalogObject.getCatalog_version()); result.addToUpdated_catalog_objects(updatedCatalogObject); result.setVersion(updatedCatalogObject.getCatalog_version()); } diff --git a/tests/custom_cluster/test_metadata_no_events_processing.py b/tests/custom_cluster/test_metadata_no_events_processing.py index fd6bc9b80..d4305e624 100644 --- a/tests/custom_cluster/test_metadata_no_events_processing.py +++ b/tests/custom_cluster/test_metadata_no_events_processing.py @@ -310,3 +310,19 @@ class TestMetadataNoEventsProcessing(CustomClusterTestSuite): assert res.data == ["i\tint\t"] finally: self.run_stmt_in_hive("drop database %s cascade" % db) + + @CustomClusterTestSuite.with_args( + catalogd_args="--catalog_topic_mode=minimal --hms_event_polling_interval_s=0", + impalad_args="--use_local_catalog=true") + def test_invalidate_metadata_v2(self, unique_name): + """Verify invalidate metadata on tables under unloaded db won't fail""" + db = unique_name + "_db" + tbl = db + "." + unique_name + "_tbl" + try: + self.run_stmt_in_hive("create database " + db) + self.run_stmt_in_hive("create table %s (i int)" % tbl) + self.client.execute("invalidate metadata %s" % tbl) + res = self.client.execute("describe %s" % tbl) + assert res.data == ["i\tint\t"] + finally: + self.run_stmt_in_hive("drop database %s cascade" % db)
