This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2107bc5fdaf [fix](row-count-cache) use cached row count for show
(#33911)
2107bc5fdaf is described below
commit 2107bc5fdaf7f623a2839e8f1820e828dc7b7185
Author: Mingyu Chen <[email protected]>
AuthorDate: Sat Apr 20 17:37:57 2024 +0800
[fix](row-count-cache) use cached row count for show (#33911)
---
.../main/java/org/apache/doris/catalog/Table.java | 5 +++++
.../java/org/apache/doris/catalog/TableIf.java | 5 +++++
.../doris/datasource/ExternalMetaCacheMgr.java | 2 +-
.../doris/datasource/ExternalRowCountCache.java | 24 ++++++++++++++++++++++
.../org/apache/doris/datasource/ExternalTable.java | 13 ++++++++++++
.../java/org/apache/doris/qe/ShowExecutor.java | 4 ++--
.../apache/doris/service/FrontendServiceImpl.java | 2 +-
7 files changed, 51 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index 52655fa0649..5d607cd2260 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -662,4 +662,9 @@ public abstract class Table extends MetaObject implements
Writable, TableIf {
public List<Pair<String, String>> getColumnIndexPairs(Set<String> columns)
{
return Lists.newArrayList();
}
+
+ @Override
+ public long getCachedRowCount() {
+ return getRowCount();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index f7c8b4b8325..1ba8ee30766 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -160,6 +160,11 @@ public interface TableIf {
long getRowCount();
+ // Get the row count from cache,
+ // If miss, just return 0
+ // This is used for external table, because for external table, the
fetching row count may be expensive
+ long getCachedRowCount();
+
long fetchRowCount();
long getDataLength();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index 6198f1bab9a..88278a1d342 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -89,7 +89,7 @@ public class ExternalMetaCacheMgr {
public ExternalMetaCacheMgr() {
rowCountRefreshExecutor = ThreadPoolManager.newDaemonFixedThreadPool(
Config.max_external_cache_loader_thread_pool_size,
- Config.max_external_cache_loader_thread_pool_size,
+ Config.max_external_cache_loader_thread_pool_size * 1000,
"RowCountRefreshExecutor", 0, true);
commonRefreshExecutor = ThreadPoolManager.newDaemonFixedThreadPool(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
index 4f88d534f92..44aecbca1d5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
@@ -114,4 +114,28 @@ public class ExternalRowCountCache {
return 0;
}
+ /**
+ * Get cached row count for the given table if present. Return 0 if cached
not loaded.
+ * This method will not trigger async loading if cache is missing.
+ *
+ * @param catalogId
+ * @param dbId
+ * @param tableId
+ * @return
+ */
+ public long getCachedRowCountIfPresent(long catalogId, long dbId, long
tableId) {
+ RowCountKey key = new RowCountKey(catalogId, dbId, tableId);
+ try {
+ CompletableFuture<Optional<Long>> f =
rowCountCache.getIfPresent(key);
+ if (f == null) {
+ return 0;
+ } else if (f.isDone()) {
+ return f.get().orElse(0L);
+ }
+ } catch (Exception e) {
+ LOG.warn("Unexpected exception while returning row count if
present", e);
+ }
+ return 0;
+ }
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 82390b91656..952b5c64cf8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -206,6 +206,19 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
return
Env.getCurrentEnv().getExtMetaCacheMgr().getRowCountCache().getCachedRowCount(catalog.getId(),
dbId, id);
}
+ @Override
+ public long getCachedRowCount() {
+ // Return 0 if makeSureInitialized throw exception.
+ // For example, init hive table may throw NotSupportedException.
+ try {
+ makeSureInitialized();
+ } catch (Exception e) {
+ LOG.warn("Failed to initialize table {}.{}.{}", catalog.getName(),
dbName, name, e);
+ return 0;
+ }
+ return
Env.getCurrentEnv().getExtMetaCacheMgr().getRowCountCache().getCachedRowCount(catalog.getId(),
dbId, id);
+ }
+
@Override
/**
* Default return 0. Subclass need to implement this interface.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 54d2f6d4ced..a77dc833931 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -986,7 +986,7 @@ public class ShowExecutor {
// Row_format
row.add(null);
// Rows
- row.add(String.valueOf(table.getRowCount()));
+ row.add(String.valueOf(table.getCachedRowCount()));
// Avg_row_length
row.add(String.valueOf(table.getAvgRowLength()));
// Data_length
@@ -2536,7 +2536,7 @@ public class ShowExecutor {
tableStats == null means it's not analyzed, in this case show the
estimated row count.
*/
if (tableStats == null) {
- resultSet =
showTableStatsStmt.constructResultSet(tableIf.getRowCount());
+ resultSet =
showTableStatsStmt.constructResultSet(tableIf.getCachedRowCount());
} else {
resultSet = showTableStatsStmt.constructResultSet(tableStats);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index fb659f8d712..08d4c0ada68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -624,7 +624,7 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
status.setUpdateTime(table.getUpdateTime() / 1000);
status.setCheckTime(lastCheckTime / 1000);
status.setCollation("utf-8");
- status.setRows(table.getRowCount());
+ status.setRows(table.getCachedRowCount());
status.setDataLength(table.getDataLength());
status.setAvgRowLength(table.getAvgRowLength());
tablesResult.add(status);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
