This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new b366cada910 [bugfix](hive)use originHiveKeys for hive partitionvalue for 2.0 (#33240) b366cada910 is described below commit b366cada910304084b9083c2afafa3cafc5c1468 Author: wuwenchi <wuwenchi...@hotmail.com> AuthorDate: Thu Apr 4 08:14:19 2024 +0800 [bugfix](hive)use originHiveKeys for hive partitionvalue for 2.0 (#33240) mirror: #32664 #32768 --- .../org/apache/doris/catalog/PartitionKey.java | 10 ++ .../doris/planner/ListPartitionPrunerV2Test.java | 135 +++++++++++++++++++++ 2 files changed, 145 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index bf69a209e9e..4e273de373c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -208,6 +208,11 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { } public List<String> getPartitionValuesAsStringList() { + if (originHiveKeys.size() == keys.size()) { + // for hive, we need ues originHiveKeys + // because when a double 1.234 as partition column, it will save as '1.123000' for PartitionValue + return getPartitionValuesAsStringListForHive(); + } return keys.stream().map(k -> k.getStringValue()).collect(Collectors.toList()); } @@ -538,4 +543,9 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { } return false; } + + // for test + public List<String> getOriginHiveKeys() { + return originHiveKeys; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java new file mode 100644 index 00000000000..e837af6839b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.planner; + +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ThreadPoolManager; +import org.apache.doris.datasource.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HiveMetaStoreCache; +import org.apache.doris.datasource.hive.PooledHiveMetaStoreClient; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import mockit.Mock; +import mockit.MockUp; +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; + +public class ListPartitionPrunerV2Test { + @Test + public void testPartitionValuesMap() throws AnalysisException { + List<PartitionValue> partitionValues = new ArrayList<>(); + partitionValues.add(new PartitionValue("1.123000")); + ArrayList<Type> types = new ArrayList<>(); + types.add(ScalarType.DOUBLE); + + // for hive table + PartitionKey key = PartitionKey.createListPartitionKeyWithTypes(partitionValues, types, true); + ListPartitionItem listPartitionItem = new ListPartitionItem(Lists.newArrayList(key)); + Map<Long, PartitionItem> idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionValues.size()); + idToPartitionItem.put(1L, listPartitionItem); + + // for olap table + PartitionKey key2 = PartitionKey.createListPartitionKeyWithTypes(partitionValues, types, false); + ListPartitionItem listPartitionItem2 = new ListPartitionItem(Lists.newArrayList(key2)); + idToPartitionItem.put(2L, listPartitionItem2); + + Map<Long, List<String>> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); + Assert.assertEquals("1.123000", partitionValuesMap.get(1L).get(0)); + Assert.assertEquals("1.123", partitionValuesMap.get(2L).get(0)); + } + + @Test + public void testInvalidateTable() { + + new MockUp<HMSExternalCatalog>(HMSExternalCatalog.class) { + @Mock + public PooledHiveMetaStoreClient getClient() { + return new PooledHiveMetaStoreClient(new HiveConf(), 2); + } + }; + + new MockUp<PooledHiveMetaStoreClient>(PooledHiveMetaStoreClient.class) { + @Mock + public List<String> listPartitionNames(String dbName, String tblName) { + // Mock is used here to represent the existence of a partition in the original table + return new ArrayList<String>() {{ + add("c1=1.234000"); + }}; + } + }; + + ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool( + 20, 20, "mgr", 120, false); + HiveMetaStoreCache cache = new HiveMetaStoreCache( + new HMSExternalCatalog(1L, "catalog", null, new HashMap<>(), null), executor); + ArrayList<Type> types = new ArrayList<>(); + types.add(ScalarType.DOUBLE); + + // test cache + // the original partition of the table (in mock) will be loaded here + String dbName = "db"; + String tblName = "tb"; + HiveMetaStoreCache.HivePartitionValues partitionValues = cache.getPartitionValues(dbName, tblName, types); + Assert.assertEquals(1, partitionValues.getIdToPartitionItem().size()); + Assert.assertTrue(partitionValues.getIdToPartitionItem().containsKey(0L)); + List<PartitionKey> items = partitionValues.getIdToPartitionItem().get(0L).getItems(); + Assert.assertEquals(1, items.size()); + PartitionKey partitionKey = items.get(0); + Assert.assertEquals("1.234", partitionKey.getKeys().get(0).toString()); + Assert.assertEquals("1.234000", partitionKey.getOriginHiveKeys().get(0)); + + // test add cache + ArrayList<String> values = new ArrayList<>(); + values.add("c1=5.678000"); + cache.addPartitionsCache(dbName, tblName, values, types); + HiveMetaStoreCache.HivePartitionValues partitionValues2 = cache.getPartitionValues(dbName, tblName, types); + Assert.assertEquals(2, partitionValues2.getIdToPartitionItem().size()); + Assert.assertTrue(partitionValues2.getIdToPartitionItem().containsKey(1L)); + List<PartitionKey> items2 = partitionValues2.getIdToPartitionItem().get(1L).getItems(); + Assert.assertEquals(1, items2.size()); + PartitionKey partitionKey2 = items2.get(0); + Assert.assertEquals("5.678", partitionKey2.getKeys().get(0).toString()); + Assert.assertEquals("5.678000", partitionKey2.getOriginHiveKeys().get(0)); + + // test refresh table + // simulates the manually added partition table being deleted, leaving only one original partition in mock + cache.invalidateTableCache(dbName, tblName); + HiveMetaStoreCache.HivePartitionValues partitionValues3 = cache.getPartitionValues(dbName, tblName, types); + Assert.assertEquals(1, partitionValues3.getIdToPartitionItem().size()); + Assert.assertTrue(partitionValues3.getIdToPartitionItem().containsKey(0L)); + List<PartitionKey> items3 = partitionValues3.getIdToPartitionItem().get(0L).getItems(); + Assert.assertEquals(1, items3.size()); + PartitionKey partitionKey3 = items3.get(0); + Assert.assertEquals("1.234", partitionKey3.getKeys().get(0).toString()); + Assert.assertEquals("1.234000", partitionKey3.getOriginHiveKeys().get(0)); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org