This is an automated email from the ASF dual-hosted git repository. boroknagyz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 20d289bd13930b29fda0d96c38ae2fff6ca7c36b Author: Zoltan Borok-Nagy <[email protected]> AuthorDate: Sat Mar 1 10:12:00 2025 +0100 IMPALA-13814: test_iceberg_table_metrics fails in non-DFS builds test_iceberg_table_metrics checks the value of memory-estimate-bytes for Iceberg tables. The value depends on the number of blocks in the table which will be 0 in non-DFS (not HDFS, not Ozone) builds. With this patch the test has different checks for filesystems that support block location information and for file filesystems that don't. Change-Id: Ie0c6d8a93e0a07cc680fac6e3f83218b853ca5e3 Reviewed-on: http://gerrit.cloudera.org:8080/22565 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../main/java/org/apache/impala/common/FileSystemUtil.java | 2 ++ tests/util/filesystem_utils.py | 6 ++++++ tests/webserver/test_web_pages.py | 13 +++++++++---- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java index 4ea9f6b78..4406d3fdc 100644 --- a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java +++ b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java @@ -92,6 +92,8 @@ public class FileSystemUtil { /** * Set containing all FileSystem scheme that known to supports storage UUIDs in * BlockLocation calls. + * Keep the list in sync with Python method + * tests/util/filesystem_utils.py::supports_storage_ids() */ private static final Set<String> SCHEME_SUPPORT_STORAGE_IDS = ImmutableSet.<String>builder() diff --git a/tests/util/filesystem_utils.py b/tests/util/filesystem_utils.py index 42d0e69e3..d81e34913 100644 --- a/tests/util/filesystem_utils.py +++ b/tests/util/filesystem_utils.py @@ -95,6 +95,12 @@ def get_fs_uri_scheme(): return uri_scheme +def supports_storage_ids(): + """Returns True if the target filesystem supports storage UUIDs in BlockLocation + calls. Keep in sync with Java method FileSystemUtil.supportsStorageIds().""" + return IS_HDFS or IS_OZONE + + WAREHOUSE = get_fs_path('/test-warehouse') FILESYSTEM_NAME = get_fs_name(FILESYSTEM) WAREHOUSE_PREFIX = os.getenv("WAREHOUSE_LOCATION_PREFIX") diff --git a/tests/webserver/test_web_pages.py b/tests/webserver/test_web_pages.py index 52c8f59fe..cac4f407a 100644 --- a/tests/webserver/test_web_pages.py +++ b/tests/webserver/test_web_pages.py @@ -21,7 +21,7 @@ from tests.common.file_utils import grep_dir from tests.common.skip import SkipIfBuildType, SkipIfDockerizedCluster from tests.common.impala_cluster import ImpalaCluster from tests.common.impala_test_suite import ImpalaTestSuite -from tests.util.filesystem_utils import IS_HDFS +from tests.util.filesystem_utils import supports_storage_ids from tests.util.parse_util import parse_duration_string_ms from datetime import datetime from multiprocessing import Process, Queue @@ -1011,11 +1011,16 @@ class TestWebPage(ImpalaTestSuite): "functional_parquet", "iceberg_non_partitioned", "total-file-size-bytes") assert '20' == self.__get_table_metric( "functional_parquet", "iceberg_non_partitioned", "num-files") - assert '13000' == self.__get_table_metric( - "functional_parquet", "iceberg_non_partitioned", "memory-estimate-bytes") - if IS_HDFS: + if supports_storage_ids(): + # Target FS has block location information. assert '20' == self.__get_table_metric( "functional_parquet", "iceberg_non_partitioned", "num-blocks") + assert '13000' == self.__get_table_metric( + "functional_parquet", "iceberg_non_partitioned", "memory-estimate-bytes") + else: + # Target FS doesn't have block locations, so 'memory-estimate-bytes' differ. + assert '10000' == self.__get_table_metric( + "functional_parquet", "iceberg_non_partitioned", "memory-estimate-bytes") def __get_table_metric(self, db_name, tbl_name, metric): self.client.execute("refresh %s.%s" % (db_name, tbl_name))
