This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-2.1-lakehouse in repository https://gitbox.apache.org/repos/asf/doris.git
commit fc6691f7ee8b89b7bba25b89deb9d7a659f400e8 Author: wuwenchi <[email protected]> AuthorDate: Mon Feb 17 10:02:24 2025 +0800 [opt](iceberg)Improve performance by not retrieving table objects for hms (#47782) ### What problem does this PR solve? When Iceberg retrieves all tables in a database, it first fetches all table names, then retrieves the corresponding table objects based on the table names, and finally determines whether a table is an Iceberg table by checking if the type attribute of the table is "iceberg". However, for lower - version HMS (Hive Metastore Service), it doesn't have the `getTableObjectsByName` method and can only use the `get_table` method. When there are 1000 tables in a database, the `get_table` method will be called 1000 times, which is extremely time consuming. Therefore, by default, the table type is not checked, and all tables are displayed. If you still want to perform this check, you can add a configuration to the catalog:`"list-all-tables"="false"` --- .../doris/datasource/iceberg/IcebergUtils.java | 6 +++ .../doris/datasource/iceberg/IcebergUtilsTest.java | 63 ++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 98eab7faba0..44d6ca28173 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -686,6 +686,12 @@ public class IcebergUtils { hiveCatalog.setConf(externalCatalog.getConfiguration()); Map<String, String> catalogProperties = externalCatalog.getProperties(); + if (!catalogProperties.containsKey(HiveCatalog.LIST_ALL_TABLES)) { + // This configuration will display all tables (including non-Iceberg type tables), + // which can save the time of obtaining table objects. + // Later, type checks will be performed when loading the table. + catalogProperties.put(HiveCatalog.LIST_ALL_TABLES, "true"); + } String metastoreUris = catalogProperties.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, ""); catalogProperties.put(CatalogProperties.URI, metastoreUris); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java new file mode 100644 index 00000000000..244f173acb7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.iceberg.hive.HiveCatalog; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.HashMap; + +public class IcebergUtilsTest { + @Test + public void testParseTableName() { + try { + IcebergHMSExternalCatalog c1 = + new IcebergHMSExternalCatalog(1, "name", null, new HashMap<>(), ""); + HiveCatalog i1 = IcebergUtils.createIcebergHiveCatalog(c1, "i1"); + Assert.assertTrue(getListAllTables(i1)); + + IcebergHMSExternalCatalog c2 = + new IcebergHMSExternalCatalog(1, "name", null, + new HashMap<String, String>() {{ + put("list-all-tables", "true"); + }}, + ""); + HiveCatalog i2 = IcebergUtils.createIcebergHiveCatalog(c2, "i1"); + Assert.assertTrue(getListAllTables(i2)); + + IcebergHMSExternalCatalog c3 = + new IcebergHMSExternalCatalog(1, "name", null, + new HashMap<String, String>() {{ + put("list-all-tables", "false"); + }}, + ""); + HiveCatalog i3 = IcebergUtils.createIcebergHiveCatalog(c3, "i1"); + Assert.assertFalse(getListAllTables(i3)); + } catch (Exception e) { + Assert.fail(); + } + } + + private boolean getListAllTables(HiveCatalog hiveCatalog) throws IllegalAccessException, NoSuchFieldException { + Field declaredField = hiveCatalog.getClass().getDeclaredField("listAllTables"); + declaredField.setAccessible(true); + return declaredField.getBoolean(hiveCatalog); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
