This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new a794109e3bb branch-4.1: [fix](hive table) Fill Hive meta cache when
loading row count for queries #63470 (#63800)
a794109e3bb is described below
commit a794109e3bb5d6e639f7ae941a1eb6fc08391e3d
Author: yujun <[email protected]>
AuthorDate: Fri May 29 14:49:34 2026 +0800
branch-4.1: [fix](hive table) Fill Hive meta cache when loading row count
for queries #63470 (#63800)
cherry-pick: #63470
---
.../doris/datasource/ExternalRowCountCache.java | 53 ++++---
.../org/apache/doris/datasource/ExternalTable.java | 13 +-
.../doris/datasource/hive/HMSExternalTable.java | 28 ++--
.../datasource/ExternalRowCountCacheTest.java | 66 +++++++-
.../datasource/hive/HMSExternalTableTest.java | 176 +++++++++++++++++++++
5 files changed, 297 insertions(+), 39 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
index 352f733f8e7..f32ba5ae20c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
@@ -83,40 +83,49 @@ public class ExternalRowCountCache {
public static class RowCountCacheLoader extends
BasicAsyncCacheLoader<RowCountKey, Optional<Long>> {
@Override
protected Optional<Long> doLoad(RowCountKey rowCountKey) {
- try {
- TableIf table =
StatisticsUtil.findTable(rowCountKey.catalogId, rowCountKey.dbId,
rowCountKey.tableId);
- return Optional.of(table.fetchRowCount());
- } catch (Exception e) {
- String message = String.format("Failed to get table row count
with catalogId %s, dbId %s, tableId %s. "
- + "Reason %s",
- rowCountKey.catalogId, rowCountKey.dbId,
rowCountKey.tableId, e.getMessage());
- if (LOG.isDebugEnabled()) {
- LOG.debug(message, e);
- } else {
- LOG.warn(message);
- }
+ return loadRowCount(rowCountKey, false);
+ }
+ }
- // Return Optional.empty() will cache this empty value in
memory,
- // so we can't try to load the row count until the cache
expire.
- // Throw an exception here will cause too much stack log in
fe.out.
- // So we return null when exception happen.
- // Null may raise NPE in caller, but that is expected.
- // We catch that NPE and return a default value -1 without
keep the value in cache,
- // so we can trigger the load function to fetch row count
again next time in this exception case.
- return null;
+ static Optional<Long> loadRowCount(RowCountKey rowCountKey, boolean
fillMetaCache) {
+ try {
+ ExternalTable table = (ExternalTable) StatisticsUtil.findTable(
+ rowCountKey.catalogId, rowCountKey.dbId,
rowCountKey.tableId);
+ return
Optional.of(table.fetchRowCountWithMetaCache(fillMetaCache));
+ } catch (Exception e) {
+ String message = String.format("Failed to get table row count with
catalogId %s, dbId %s, tableId %s. "
+ + "Reason %s",
+ rowCountKey.catalogId, rowCountKey.dbId,
rowCountKey.tableId, e.getMessage());
+ if (LOG.isDebugEnabled()) {
+ LOG.warn(message, e);
+ } else {
+ LOG.warn(message);
}
+
+ // Return Optional.empty() will cache this empty value in memory,
+ // so we can't try to load the row count until the cache expire.
+ // Throw an exception here will cause too much stack log in fe.out.
+ // So we return null when exception happen.
+ // Null may raise NPE in caller, but that is expected.
+ // We catch that NPE and return a default value -1 without keep
the value in cache,
+ // so we can trigger the load function to fetch row count again
next time in this exception case.
+ return null;
}
}
/**
* Get cached row count for the given table. Return -1 if cached not
loaded or table not exists.
* Cached will be loaded async.
+ * @param fillMetaCache whether loading the row count may fill external
metadata caches
* @return Cached row count or -1 if not exist
*/
- public long getCachedRowCount(long catalogId, long dbId, long tableId) {
+ public long getCachedRowCount(long catalogId, long dbId, long tableId,
boolean fillMetaCache) {
RowCountKey key = new RowCountKey(catalogId, dbId, tableId);
try {
- CompletableFuture<Optional<Long>> f = rowCountCache.get(key);
+ CompletableFuture<Optional<Long>> f = fillMetaCache
+ ? rowCountCache.get(key, (rowCountKey, executor) ->
CompletableFuture.supplyAsync(
+ () -> loadRowCount(rowCountKey, true), executor))
+ : rowCountCache.get(key);
// Get row count synchronously by default.
if (ConnectContext.get() == null
||
ConnectContext.get().getSessionVariable().fetchHiveRowCountSync) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 800f98cea77..f5786423b6e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -251,7 +251,8 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
return TableIf.UNKNOWN_ROW_COUNT;
}
// All external table should get external row count from cache.
- return
Env.getCurrentEnv().getExtMetaCacheMgr().getRowCountCache().getCachedRowCount(catalog.getId(),
dbId, id);
+ return Env.getCurrentEnv().getExtMetaCacheMgr().getRowCountCache()
+ .getCachedRowCount(catalog.getId(), dbId, id, true);
}
@Override
@@ -265,7 +266,8 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
}
// getExtMetaCacheMgr().getRowCountCache().getCachedRowCount() is an
asynchronous non-blocking operation.
// For tables that are not in the cache, it will load asynchronously
and return -1.
- return
Env.getCurrentEnv().getExtMetaCacheMgr().getRowCountCache().getCachedRowCount(catalog.getId(),
dbId, id);
+ return Env.getCurrentEnv().getExtMetaCacheMgr().getRowCountCache()
+ .getCachedRowCount(catalog.getId(), dbId, id, false);
}
@Override
@@ -277,6 +279,13 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
return UNKNOWN_ROW_COUNT;
}
+ /**
+ * Fetch row count, and allow the load path to fill external metadata
cache if supported.
+ */
+ public long fetchRowCountWithMetaCache(boolean fillMetaCache) {
+ return fetchRowCount();
+ }
+
@Override
public long getAvgRowLength() {
return 0;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 137ee718741..b7add486018 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -776,13 +776,22 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
@Override
public long fetchRowCount() {
+ return fetchRowCountInternal(false);
+ }
+
+ @Override
+ public long fetchRowCountWithMetaCache(boolean fillMetaCache) {
+ return fetchRowCountInternal(fillMetaCache);
+ }
+
+ private long fetchRowCountInternal(boolean fillMetaCache) {
makeSureInitialized();
// Get row count from hive metastore property.
long rowCount = getRowCountFromExternalSource();
// Only hive table supports estimate row count by listing file.
if (rowCount == UNKNOWN_ROW_COUNT && dlaType.equals(DLAType.HIVE)) {
LOG.info("Will estimate row count for table {} from file list.",
name);
- rowCount = getRowCountFromFileList();
+ rowCount = getRowCountFromFileList(fillMetaCache);
}
return rowCount;
}
@@ -937,7 +946,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
@Override
public List<Long> getChunkSizes() {
HiveExternalMetaCache.HivePartitionValues partitionValues =
getAllPartitionValues();
- List<HiveExternalMetaCache.FileCacheValue> filesByPartitions =
getFilesForPartitions(partitionValues, 0);
+ List<HiveExternalMetaCache.FileCacheValue> filesByPartitions =
getFilesForPartitions(partitionValues, 0, false);
List<Long> result = Lists.newArrayList();
for (HiveExternalMetaCache.FileCacheValue files : filesByPartitions) {
for (HiveExternalMetaCache.HiveFileStatus file : files.getFiles())
{
@@ -1041,7 +1050,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
/**
* Estimate hive table row count : totalFileSize/estimatedRowSize
*/
- private long getRowCountFromFileList() {
+ private long getRowCountFromFileList(boolean fillMetaCache) {
if (!GlobalVariable.enable_get_row_count_from_file_list) {
return UNKNOWN_ROW_COUNT;
}
@@ -1055,7 +1064,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
// Get files for all partitions.
int samplePartitionSize = Config.hive_stats_partition_sample_size;
List<HiveExternalMetaCache.FileCacheValue> filesByPartitions =
- getFilesForPartitions(partitionValues,
samplePartitionSize);
+ getFilesForPartitions(partitionValues,
samplePartitionSize, fillMetaCache);
LOG.info("Number of files selected for hive table {} is {}", name,
filesByPartitions.size());
long totalSize = 0;
// Calculate the total file size.
@@ -1121,7 +1130,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
// Get all files related to given partition values
// If sampleSize > 0, randomly choose part of partitions of the whole
table.
private List<HiveExternalMetaCache.FileCacheValue> getFilesForPartitions(
- HiveExternalMetaCache.HivePartitionValues partitionValues, int
sampleSize) {
+ HiveExternalMetaCache.HivePartitionValues partitionValues, int
sampleSize, boolean fillMetaCache) {
if (isView()) {
return Lists.newArrayList();
}
@@ -1146,9 +1155,10 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
for (PartitionItem item : partitionItems) {
partitionValuesList.add(((ListPartitionItem)
item).getItems().get(0).getPartitionValuesAsStringList());
}
- // get partitions without cache, so that it will not invalid the
cache when executing
- // non query request such as `show table status`
- hivePartitions = cache.getAllPartitionsWithoutCache(this,
partitionValuesList);
+ // Non-query requests such as `show table status` should not fill
heavy metadata caches.
+ hivePartitions = fillMetaCache
+ ? cache.getAllPartitionsWithCache(this,
partitionValuesList)
+ : cache.getAllPartitionsWithoutCache(this,
partitionValuesList);
LOG.info("Partition list size for hive partition table {} is {}",
name, hivePartitions.size());
} else {
hivePartitions.add(new HivePartition(getOrBuildNameMapping(), true,
@@ -1161,7 +1171,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
LOG.debug("Chosen partition for table {}. [{}]", name,
partition.toString());
}
}
- return cache.getFilesByPartitions(hivePartitions, false, true, new
FileSystemDirectoryLister(), null);
+ return cache.getFilesByPartitions(hivePartitions, fillMetaCache, true,
new FileSystemDirectoryLister(), null);
}
@Override
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java
index aac07d2d747..075806aa82b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java
@@ -19,17 +19,71 @@ package org.apache.doris.datasource;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.ThreadPoolManager;
+import org.apache.doris.statistics.util.StatisticsUtil;
+import com.google.common.util.concurrent.MoreExecutors;
import mockit.Mock;
import mockit.MockUp;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
import java.util.Optional;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;
public class ExternalRowCountCacheTest {
+ @Test
+ public void testRowCountKeyUsesTableIdAsCacheIdentity() {
+ ExternalRowCountCache.RowCountKey key1 = new
ExternalRowCountCache.RowCountKey(1, 2, 3);
+ ExternalRowCountCache.RowCountKey key2 = new
ExternalRowCountCache.RowCountKey(2, 3, 3);
+
+ Assertions.assertEquals(key1, key2);
+ Assertions.assertEquals(key1.hashCode(), key2.hashCode());
+ }
+
+ @Test
+ public void testLoadRowCountPassesFillMetaCacheToTable() {
+ ExternalTable table = Mockito.mock(ExternalTable.class);
+ Mockito.when(table.fetchRowCountWithMetaCache(true)).thenReturn(100L);
+ Mockito.when(table.fetchRowCountWithMetaCache(false)).thenReturn(200L);
+
+ new MockUp<StatisticsUtil>() {
+ @Mock
+ public TableIf findTable(long catalogId, long dbId, long tblId) {
+ return table;
+ }
+ };
+
+ ExternalRowCountCache.RowCountKey key = new
ExternalRowCountCache.RowCountKey(1, 2, 3);
+ Assertions.assertEquals(100L, ExternalRowCountCache.loadRowCount(key,
true).get());
+ Assertions.assertEquals(200L, ExternalRowCountCache.loadRowCount(key,
false).get());
+
+ Mockito.verify(table).fetchRowCountWithMetaCache(true);
+ Mockito.verify(table).fetchRowCountWithMetaCache(false);
+ }
+
+ @Test
+ public void testGetCachedRowCountPassesFillMetaCacheToLoader() {
+ ExternalTable table = Mockito.mock(ExternalTable.class);
+ Mockito.when(table.fetchRowCountWithMetaCache(true)).thenReturn(100L);
+ Mockito.when(table.fetchRowCountWithMetaCache(false)).thenReturn(200L);
+
+ new MockUp<StatisticsUtil>() {
+ @Mock
+ public TableIf findTable(long catalogId, long dbId, long tblId) {
+ return table;
+ }
+ };
+
+ ExternalRowCountCache cache = new
ExternalRowCountCache(MoreExecutors.newDirectExecutorService());
+ Assertions.assertEquals(100L, cache.getCachedRowCount(1, 2, 3, true));
+ Assertions.assertEquals(200L, cache.getCachedRowCount(1, 2, 4, false));
+
+ Mockito.verify(table).fetchRowCountWithMetaCache(true);
+ Mockito.verify(table).fetchRowCountWithMetaCache(false);
+ }
+
@Test
public void testLoadWithException() throws Exception {
ThreadPoolExecutor executor =
ThreadPoolManager.newDaemonFixedThreadPool(
@@ -44,7 +98,7 @@ public class ExternalRowCountCacheTest {
}
};
ExternalRowCountCache cache = new ExternalRowCountCache(executor);
- long cachedRowCount = cache.getCachedRowCount(1, 1, 1);
+ long cachedRowCount = cache.getCachedRowCount(1, 1, 1, false);
Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cachedRowCount);
for (int i = 0; i < 60; i++) {
if (counter.get() == 1) {
@@ -61,16 +115,16 @@ public class ExternalRowCountCacheTest {
return Optional.of(100L);
}
};
- cache.getCachedRowCount(1, 1, 1);
+ cache.getCachedRowCount(1, 1, 1, false);
for (int i = 0; i < 60; i++) {
- cachedRowCount = cache.getCachedRowCount(1, 1, 1);
+ cachedRowCount = cache.getCachedRowCount(1, 1, 1, false);
if (cachedRowCount != TableIf.UNKNOWN_ROW_COUNT) {
Assertions.assertEquals(100, cachedRowCount);
break;
}
Thread.sleep(1000);
}
- cachedRowCount = cache.getCachedRowCount(1, 1, 1);
+ cachedRowCount = cache.getCachedRowCount(1, 1, 1, false);
Assertions.assertEquals(100, cachedRowCount);
Assertions.assertEquals(2, counter.get());
@@ -86,10 +140,10 @@ public class ExternalRowCountCacheTest {
return Optional.of(100L);
}
};
- cachedRowCount = cache.getCachedRowCount(2, 2, 2);
+ cachedRowCount = cache.getCachedRowCount(2, 2, 2, false);
Assertions.assertEquals(100, cachedRowCount);
Thread.sleep(1000);
- cachedRowCount = cache.getCachedRowCount(2, 2, 2);
+ cachedRowCount = cache.getCachedRowCount(2, 2, 2, false);
Assertions.assertEquals(100, cachedRowCount);
for (int i = 0; i < 60; i++) {
if (counter.get() == 3) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java
index bd29766f39e..e68d3704a0c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java
@@ -17,10 +17,32 @@
package org.apache.doris.datasource.hive;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ListPartitionItem;
+import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.catalog.PartitionKey;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.jmockit.Deencapsulation;
+import org.apache.doris.datasource.ExternalMetaCacheMgr;
+import org.apache.doris.fs.FileSystemDirectoryLister;
+
+import com.google.common.collect.HashBiMap;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
import mockit.Injectable;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
/**
@@ -88,6 +110,160 @@ public class HMSExternalTableTest {
Assertions.assertEquals(TEST_VIEW_TEXT, table.getViewText());
}
+ private Table buildRemoteTableWithInputFormat(String inputFormatName) {
+ SerDeInfo serDeInfo = new SerDeInfo();
+
serDeInfo.setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setInputFormat(inputFormatName);
+ sd.setSerdeInfo(serDeInfo);
+ Table remoteTable = new Table();
+ remoteTable.setSd(sd);
+ return remoteTable;
+ }
+
+ @Test
+ public void testFetchRowCountFillsMetaCacheOnlyWhenRequested() throws
Exception {
+ long catalogId = 100L;
+ String localDbName = "test_db";
+ String partitionValue = "2026-05-21";
+ String inputFormat = "org.apache.hadoop.mapred.TextInputFormat";
+ String partitionLocation =
"file:///tmp/doris_hms_row_count_cache/dt=2026-05-21";
+
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ HMSExternalDatabase db = Mockito.mock(HMSExternalDatabase.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("test_catalog");
+ Mockito.when(catalog.getProperties()).thenReturn(ImmutableMap.of());
+ Mockito.when(db.getFullName()).thenReturn(localDbName);
+
+ Table remoteTable = buildRemoteTableWithInputFormat(inputFormat);
+ remoteTable.setParameters(ImmutableMap.of());
+ TestHMSExternalTableForMetaCache table = new
TestHMSExternalTableForMetaCache(
+ catalog, db, remoteTable, partitionValue);
+ Deencapsulation.setField(table, "dlaType",
HMSExternalTable.DLAType.HIVE);
+
+ List<HivePartition> partitions = Collections.singletonList(new
HivePartition(
+ null, false, inputFormat, partitionLocation,
Collections.singletonList(partitionValue),
+ Collections.emptyMap()));
+ HiveExternalMetaCache.FileCacheValue fileCacheValue = new
HiveExternalMetaCache.FileCacheValue();
+ HiveExternalMetaCache.HiveFileStatus status = new
HiveExternalMetaCache.HiveFileStatus();
+ status.setLength(128L);
+ fileCacheValue.getFiles().add(status);
+ List<HiveExternalMetaCache.FileCacheValue> files =
Collections.singletonList(fileCacheValue);
+
+ HiveExternalMetaCache hiveCache =
Mockito.mock(HiveExternalMetaCache.class);
+ Mockito.when(hiveCache.getAllPartitionsWithCache(Mockito.eq(table),
Mockito.anyList()))
+ .thenReturn(partitions);
+ Mockito.when(hiveCache.getAllPartitionsWithoutCache(Mockito.eq(table),
Mockito.anyList()))
+ .thenReturn(partitions);
+ Mockito.when(hiveCache.getFilesByPartitions(Mockito.eq(partitions),
Mockito.eq(true), Mockito.eq(true),
+ Mockito.any(FileSystemDirectoryLister.class),
Mockito.isNull()))
+ .thenReturn(files);
+ Mockito.when(hiveCache.getFilesByPartitions(Mockito.eq(partitions),
Mockito.eq(false), Mockito.eq(true),
+ Mockito.any(FileSystemDirectoryLister.class),
Mockito.isNull()))
+ .thenReturn(files);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr extMetaCacheMgr =
Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(extMetaCacheMgr);
+ Mockito.when(extMetaCacheMgr.hive(catalogId)).thenReturn(hiveCache);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+
+ Assertions.assertEquals(32L,
table.fetchRowCountWithMetaCache(true));
+
Mockito.verify(hiveCache).getAllPartitionsWithCache(Mockito.eq(table),
Mockito.anyList());
+ Mockito.verify(hiveCache, Mockito.never())
+ .getAllPartitionsWithoutCache(Mockito.eq(table),
Mockito.anyList());
+
Mockito.verify(hiveCache).getFilesByPartitions(Mockito.eq(partitions),
Mockito.eq(true),
+ Mockito.eq(true),
Mockito.any(FileSystemDirectoryLister.class), Mockito.isNull());
+
+ Mockito.clearInvocations(hiveCache);
+
+ Assertions.assertEquals(32L, table.fetchRowCount());
+
Mockito.verify(hiveCache).getAllPartitionsWithoutCache(Mockito.eq(table),
Mockito.anyList());
+ Mockito.verify(hiveCache, Mockito.never())
+ .getAllPartitionsWithCache(Mockito.eq(table),
Mockito.anyList());
+
Mockito.verify(hiveCache).getFilesByPartitions(Mockito.eq(partitions),
Mockito.eq(false),
+ Mockito.eq(true),
Mockito.any(FileSystemDirectoryLister.class), Mockito.isNull());
+ }
+ }
+
+ private static class TestHMSExternalTableWithRemote extends
HMSExternalTable {
+ private final Table remoteTable;
+
+ public TestHMSExternalTableWithRemote(HMSExternalCatalog catalog,
+ HMSExternalDatabase db, Table remoteTable) {
+ super(1L, "test_table", "test_table", catalog, db);
+ this.remoteTable = remoteTable;
+ }
+
+ @Override
+ public Table getRemoteTable() {
+ return remoteTable;
+ }
+
+ @Override
+ protected synchronized void makeSureInitialized() {
+ this.objectCreated = true;
+ }
+ }
+
+ private static class TestHMSExternalTableForMetaCache extends
TestHMSExternalTableWithRemote {
+ private final Column dataColumn = new Column("c1", Type.INT);
+ private final Column partitionColumn = new Column("dt", Type.VARCHAR);
+ private final HiveExternalMetaCache.HivePartitionValues
partitionValues;
+
+ public TestHMSExternalTableForMetaCache(HMSExternalCatalog catalog,
HMSExternalDatabase db,
+ Table remoteTable, String partitionValue) throws Exception {
+ super(catalog, db, remoteTable);
+ PartitionKey partitionKey =
PartitionKey.createListPartitionKeyWithTypes(
+ Lists.newArrayList(new
org.apache.doris.analysis.PartitionValue(partitionValue)),
+ Lists.newArrayList(Type.VARCHAR),
+ true);
+ PartitionItem partitionItem = new
ListPartitionItem(Lists.newArrayList(partitionKey));
+ long partitionId = 1L;
+ Map<Long, PartitionItem> idToPartitionItem =
ImmutableMap.of(partitionId, partitionItem);
+ this.partitionValues = new
HiveExternalMetaCache.HivePartitionValues(
+ idToPartitionItem,
+ HashBiMap.create(ImmutableMap.of("dt=" + partitionValue,
partitionId)),
+ ImmutableMap.of(partitionId,
Collections.singletonList(partitionValue)));
+ }
+
+ @Override
+ public List<Column> getFullSchema() {
+ return Lists.newArrayList(dataColumn, partitionColumn);
+ }
+
+ @Override
+ public boolean isView() {
+ return false;
+ }
+
+ @Override
+ public List<Type>
getPartitionColumnTypes(java.util.Optional<org.apache.doris.datasource.mvcc.MvccSnapshot>
+ snapshot) {
+ return Collections.singletonList(Type.VARCHAR);
+ }
+
+ @Override
+ public List<Column> getPartitionColumns() {
+ return Collections.singletonList(partitionColumn);
+ }
+
+ @Override
+ public List<Column>
getPartitionColumns(java.util.Optional<org.apache.doris.datasource.mvcc.MvccSnapshot>
+ snapshot) {
+ return Collections.singletonList(partitionColumn);
+ }
+
+ @Override
+ public HiveExternalMetaCache.HivePartitionValues
getHivePartitionValues(
+
java.util.Optional<org.apache.doris.datasource.mvcc.MvccSnapshot> snapshot) {
+ return partitionValues;
+ }
+ }
+
/**
* Test implementation of HMSExternalTable that allows setting view texts
* Uses parent's getViewText() implementation for actual testing
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
