This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit bd1d613b876ee910cd2a2abe1e62e7e35257928f Author: Sai Hemanth Gantasala <[email protected]> AuthorDate: Tue Jun 13 10:32:37 2023 -0700 IMPALA-12210: CatalogD should ignore validWriteId List if the table is external while loading a table In external frontend mode, running queries on external tables shows an exception in the catalogd.INFO log and catalogd forwards the get_table_req() requests to HMS. For external tables, we don't need to verify the validWriteIdList. So a transactional check on the table is added in CatalogD before checking for compactionId on the table. By doing so, when the table is loaded, catalogD returns the table instead of forwarding the request to HMS. Testing: Manually verified that exception is not seen in the logs. Also, repeated execution of query, the request is not forwarded to HMS and instead returned from catalogD's cache. Testing with TPCH queries resulted in query performance improvement of ~ 0.04 to 0.07 secs. Change-Id: I28a34185c788a39aa902b0cd1ed0ba0c8c376ff1 Reviewed-on: http://gerrit.cloudera.org:8080/20066 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Quanlong Huang <[email protected]> --- .../impala/catalog/CatalogServiceCatalog.java | 34 +++++++++++++++------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java index bdaff8a31..1e7ab5ac5 100644 --- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java +++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java @@ -2229,7 +2229,11 @@ public class CatalogServiceCatalog extends Catalog { LOG.trace("table {} exits in cache, last synced id {}", tbl.getFullName(), tbl.getLastSyncedEventId()); // if no validWriteIdList is provided, we return the tbl if its loaded - if (tbl.isLoaded() && validWriteIdList == null) { + // In the external front end use case it is possible that an external table might + // have validWriteIdList, so we can simply ignore this value if table is external + if (tbl.isLoaded() && (validWriteIdList == null || + (!AcidUtils.isTransactionalTable(tbl.getMetaStoreTable().getParameters())))) { + incrementCatalogDCacheHitMetric(reason); LOG.trace("returning already loaded table {}", tbl.getFullName()); return tbl; } @@ -2241,16 +2245,7 @@ public class CatalogServiceCatalog extends Catalog { // the ValidWriteIdList only when the table id matches. if (tbl instanceof HdfsTable && AcidUtils.compare((HdfsTable) tbl, validWriteIdList, tableId) >= 0) { - CatalogMonitor.INSTANCE.getCatalogdHmsCacheMetrics() - .getCounter(CatalogHmsUtils.CATALOGD_CACHE_HIT_METRIC) - .inc(); - // Update the cache stats for a HMS API from which the current method got invoked. - if (HmsApiNameEnum.contains(reason)) { - CatalogMonitor.INSTANCE.getCatalogdHmsCacheMetrics() - .getCounter(String - .format(CatalogHmsUtils.CATALOGD_CACHE_API_HIT_METRIC, reason)) - .inc(); - } + incrementCatalogDCacheHitMetric(reason); // Check if any partition of the table has a newly compacted file. // We just take the read lock here so that we don't serialize all the getTable // calls for the same table. If there are concurrent calls, it is possible we @@ -2302,6 +2297,23 @@ public class CatalogServiceCatalog extends Catalog { } } + /** + * Increments catalogD's cache hit metrics + * @param reason + */ + private void incrementCatalogDCacheHitMetric(String reason) { + CatalogMonitor.INSTANCE.getCatalogdHmsCacheMetrics() + .getCounter(CatalogHmsUtils.CATALOGD_CACHE_HIT_METRIC) + .inc(); + // Update the cache stats for a HMS API from which the current method got invoked. + if (HmsApiNameEnum.contains(reason)) { + CatalogMonitor.INSTANCE.getCatalogdHmsCacheMetrics() + .getCounter(String + .format(CatalogHmsUtils.CATALOGD_CACHE_API_HIT_METRIC, reason)) + .inc(); + } + } + /** * Replaces an existing Table with a new value if it exists and has not changed * (has the same catalog version as 'expectedCatalogVersion'). There is one exception
