(impala) branch master updated: IMPALA-14062: Adds missing timeline items in constructing PartitionDeltaUpdater

[stigahuang]

This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git


The following commit(s) were added to refs/heads/master by this push:
     new dd2d44492 IMPALA-14062: Adds missing timeline items in constructing 
PartitionDeltaUpdater
dd2d44492 is described below

commit dd2d44492d19e57985473e8974b859578ba3fea7
Author: stiga-huang <huangquanl...@gmail.com>
AuthorDate: Mon May 19 20:38:58 2025 +0800

    IMPALA-14062: Adds missing timeline items in constructing 
PartitionDeltaUpdater
    
    PartitionDeltaUpdater has two sub-classes, PartNameBasedDeltaUpdater and
    PartBasedDeltaUpdater. They are used in reloading metadata of a table.
    Their constructors invoke HMS RPCs which could be slow and should be
    tracked in the catalog timeline.
    
    This patch adds missing timeline items for those HMS RPCs.
    
    Tests:
     - Added e2e tests
    
    Change-Id: Id231c2b15869aac2dae3258817954abf119da802
    Reviewed-on: http://gerrit.cloudera.org:8080/22917
    Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
    Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
---
 fe/src/main/java/org/apache/impala/catalog/HdfsTable.java |  3 +++
 tests/query_test/test_observability.py                    | 15 +++++++++------
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java 
b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 15387f1fd..d1618f162 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -1621,10 +1621,12 @@ public class HdfsTable extends Table implements 
FeFsTable {
         partitionList = MetaStoreUtil
             .fetchPartitionsByName(client, 
Lists.newArrayList(partitionsToUpdate),
                 msTable_);
+        catalogTimeline.markEvent("Fetched updated partitions");
       } else {
         partitionList =
             MetaStoreUtil.fetchAllPartitions(
                 client_, msTable_, NUM_PARTITION_FETCH_RETRIES);
+        catalogTimeline.markEvent("Fetched all partitions");
       }
       LOG.debug("Time taken to fetch all partitions of table {}: {} msec", 
getFullName(),
           sw.stop().elapsed(TimeUnit.MILLISECONDS));
@@ -1726,6 +1728,7 @@ public class HdfsTable extends Table implements FeFsTable 
{
       // (~.3 secs for 30K partitions).
       partitionNamesFromHms_ = new HashSet<>(client_
           .listPartitionNames(db_.getName(), name_, (short) -1));
+      catalogTimeline.markEvent("Fetched partition names");
     }
 
     @Override
diff --git a/tests/query_test/test_observability.py 
b/tests/query_test/test_observability.py
index 304563deb..eb1533ab2 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -617,12 +617,15 @@ class TestObservability(ImpalaTestSuite):
     ])
     # REFRESH
     stmt = "refresh %s.t1" % unique_database
-    self.__verify_event_labels_in_profile(stmt, [
-      "Got Metastore client",
-      "Fetched table from Metastore",
-      "Loaded file metadata for 2 partitions",
-      "Finished resetMetadata request"
-    ])
+    for i in (0, 1):
+      self.client.execute("set refresh_updated_hms_partitions=" + str(i))
+      self.__verify_event_labels_in_profile(stmt, [
+        "Got Metastore client",
+        "Fetched table from Metastore",
+        "Fetched partition names" if i == 0 else "Fetched all partitions",
+        "Loaded file metadata for 2 partitions",
+        "Finished resetMetadata request"
+      ])
     # Drop table
     stmt = "drop table %s.t1" % unique_database
     self.__verify_event_labels_in_profile(stmt, [