This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch opt_dict_perf
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/opt_dict_perf by this push:
new e1fe9b1937 remove lock (#17834)
e1fe9b1937 is described below
commit e1fe9b19378fc66dad624b82e262fd5cb26f7d78
Author: AKIRA <[email protected]>
AuthorDate: Thu Mar 16 09:57:24 2023 +0900
remove lock (#17834)
---
.../doris/statistics/StatisticsCacheLoader.java | 115 +++++++++++----------
1 file changed, 58 insertions(+), 57 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java
index 08781e5689..a50d9f0274 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java
@@ -33,8 +33,13 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executor;
+/**
+ * Use to load stats cache.
+ */
public class StatisticsCacheLoader implements
AsyncCacheLoader<StatisticsCacheKey, Statistic> {
private static final Logger LOG =
LogManager.getLogger(StatisticsCacheLoader.class);
@@ -47,72 +52,68 @@ public class StatisticsCacheLoader implements
AsyncCacheLoader<StatisticsCacheKe
+ "." + StatisticConstants.HISTOGRAM_TBL_NAME + " WHERE "
+ "id = CONCAT('${tblId}', '-', ${idxId}, '-', '${colId}')";
- private static int CUR_RUNNING_LOAD = 0;
-
- private static final Object LOCK = new Object();
+ private final ConcurrentMap<StatisticsCacheKey,
CompletableFuture<Statistic>>
+ inProgressing = new ConcurrentHashMap<>();
// TODO: Maybe we should trigger a analyze job when the required
ColumnStatistic doesn't exists.
@Override
public @NonNull CompletableFuture<Statistic> asyncLoad(@NonNull
StatisticsCacheKey key,
@NonNull Executor executor) {
- synchronized (LOCK) {
- if (CUR_RUNNING_LOAD > StatisticConstants.LOAD_TASK_LIMITS) {
+ CompletableFuture<Statistic> future = inProgressing.get(key);
+ if (future != null) {
+ return future;
+ }
+ future = CompletableFuture.supplyAsync(() -> {
+ Statistic statistic = new Statistic();
+ long startTime = 0;
+ try {
+ LOG.info("Query BE for column stats:{}-{} start time:{}",
key.tableId, key.colName,
+ startTime);
+ Map<String, String> params = new HashMap<>();
+ params.put("tblId", String.valueOf(key.tableId));
+ params.put("idxId", String.valueOf(key.idxId));
+ params.put("colId", String.valueOf(key.colName));
+
+ List<ColumnStatistic> columnStatistics;
+ List<ResultRow> columnResult =
+ StatisticsUtil.execStatisticQuery(new
StringSubstitutor(params)
+ .replace(QUERY_COLUMN_STATISTICS));
try {
- LOCK.wait();
- } catch (InterruptedException e) {
- LOG.warn("Ignore interruption", e);
+ columnStatistics =
StatisticsUtil.deserializeToColumnStatistics(columnResult);
+ } catch (Exception e) {
+ LOG.warn("Failed to deserialize column statistics", e);
+ throw new CompletionException(e);
+ }
+ if (CollectionUtils.isEmpty(columnStatistics)) {
+ statistic.setColumnStatistic(ColumnStatistic.DEFAULT);
+ } else {
+ statistic.setColumnStatistic(columnStatistics.get(0));
}
- }
- CUR_RUNNING_LOAD++;
- return CompletableFuture.supplyAsync(() -> {
- Statistic statistic = new Statistic();
+ List<Histogram> histogramStatistics;
+ List<ResultRow> histogramResult =
+ StatisticsUtil.execStatisticQuery(new
StringSubstitutor(params)
+ .replace(QUERY_HISTOGRAM_STATISTICS));
try {
- Map<String, String> params = new HashMap<>();
- params.put("tblId", String.valueOf(key.tableId));
- params.put("idxId", String.valueOf(key.idxId));
- params.put("colId", String.valueOf(key.colName));
-
- List<ColumnStatistic> columnStatistics;
- List<ResultRow> columnResult =
- StatisticsUtil.execStatisticQuery(new
StringSubstitutor(params)
- .replace(QUERY_COLUMN_STATISTICS));
- try {
- columnStatistics =
StatisticsUtil.deserializeToColumnStatistics(columnResult);
- } catch (Exception e) {
- LOG.warn("Failed to deserialize column statistics", e);
- throw new CompletionException(e);
- }
- if (CollectionUtils.isEmpty(columnStatistics)) {
- statistic.setColumnStatistic(ColumnStatistic.DEFAULT);
- } else {
- statistic.setColumnStatistic(columnStatistics.get(0));
- }
-
- List<Histogram> histogramStatistics;
- List<ResultRow> histogramResult =
- StatisticsUtil.execStatisticQuery(new
StringSubstitutor(params)
- .replace(QUERY_HISTOGRAM_STATISTICS));
- try {
- histogramStatistics =
StatisticsUtil.deserializeToHistogramStatistics(histogramResult);
- } catch (Exception e) {
- LOG.warn("Failed to deserialize histogram statistics",
e);
- throw new CompletionException(e);
- }
- if (CollectionUtils.isEmpty(histogramStatistics)) {
- statistic.setHistogram(Histogram.DEFAULT);
- } else {
- statistic.setHistogram(histogramStatistics.get(0));
- }
- } finally {
- synchronized (LOCK) {
- CUR_RUNNING_LOAD--;
- LOCK.notify();
- }
+ histogramStatistics =
StatisticsUtil.deserializeToHistogramStatistics(histogramResult);
+ } catch (Exception e) {
+ LOG.warn("Failed to deserialize histogram statistics", e);
+ throw new CompletionException(e);
}
-
- return statistic;
- });
- }
+ if (CollectionUtils.isEmpty(histogramStatistics)) {
+ statistic.setHistogram(Histogram.DEFAULT);
+ } else {
+ statistic.setHistogram(histogramStatistics.get(0));
+ }
+ } finally {
+ long endTime = System.currentTimeMillis();
+ LOG.info("Query BE for column stats:{}-{} end time:{} cost
time:{}", key.tableId, key.colName,
+ endTime, endTime - startTime);
+ inProgressing.remove(key);
+ }
+ return statistic;
+ });
+ inProgressing.put(key, future);
+ return future;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
