kezhenxu94 commented on a change in pull request #7153:
URL: https://github.com/apache/skywalking/pull/7153#discussion_r656026055
##########
File path:
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/PersistenceTimer.java
##########
@@ -63,98 +70,150 @@ public void start(ModuleManager moduleManager,
CoreModuleConfig moduleConfig) {
IBatchDAO batchDAO =
moduleManager.find(StorageModule.NAME).provider().getService(IBatchDAO.class);
MetricsCreator metricsCreator =
moduleManager.find(TelemetryModule.NAME)
- .provider()
-
.getService(MetricsCreator.class);
+ .provider()
+ .getService(MetricsCreator.class);
errorCounter = metricsCreator.createCounter(
- "persistence_timer_bulk_error_count", "Error execution of the
prepare stage in persistence timer",
- MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ "persistence_timer_bulk_error_count", "Error execution of the
prepare stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
);
prepareLatency = metricsCreator.createHistogramMetric(
- "persistence_timer_bulk_prepare_latency", "Latency of the prepare
stage in persistence timer",
- MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ "persistence_timer_bulk_prepare_latency", "Latency of the
prepare stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
);
executeLatency = metricsCreator.createHistogramMetric(
- "persistence_timer_bulk_execute_latency", "Latency of the execute
stage in persistence timer",
- MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ "persistence_timer_bulk_execute_latency", "Latency of the
execute stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ );
Review comment:
Please import our code style settings to avoid these unnecessary changes
##########
File path:
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/PersistenceTimer.java
##########
@@ -63,98 +70,150 @@ public void start(ModuleManager moduleManager,
CoreModuleConfig moduleConfig) {
IBatchDAO batchDAO =
moduleManager.find(StorageModule.NAME).provider().getService(IBatchDAO.class);
MetricsCreator metricsCreator =
moduleManager.find(TelemetryModule.NAME)
- .provider()
-
.getService(MetricsCreator.class);
+ .provider()
+ .getService(MetricsCreator.class);
errorCounter = metricsCreator.createCounter(
- "persistence_timer_bulk_error_count", "Error execution of the
prepare stage in persistence timer",
- MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ "persistence_timer_bulk_error_count", "Error execution of the
prepare stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
);
prepareLatency = metricsCreator.createHistogramMetric(
- "persistence_timer_bulk_prepare_latency", "Latency of the prepare
stage in persistence timer",
- MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ "persistence_timer_bulk_prepare_latency", "Latency of the
prepare stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
);
executeLatency = metricsCreator.createHistogramMetric(
- "persistence_timer_bulk_execute_latency", "Latency of the execute
stage in persistence timer",
- MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ "persistence_timer_bulk_execute_latency", "Latency of the
execute stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ );
+ allLatency = metricsCreator.createHistogramMetric(
+ "persistence_timer_bulk_all_latency", "Latency of the all
stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
);
+
syncOperationThreadsNum = moduleConfig.getSyncThreads();
maxSyncoperationNum = moduleConfig.getMaxSyncOperationNum();
+ batchExecutorService = Executors.newSingleThreadExecutor();
executorService =
Executors.newFixedThreadPool(syncOperationThreadsNum);
+ prepareExecutorService =
Executors.newFixedThreadPool(moduleConfig.getPrepareThreads());
if (!isStarted) {
Executors.newSingleThreadScheduledExecutor()
- .scheduleWithFixedDelay(
- new RunnableWithExceptionProtection(() ->
extractDataAndSave(batchDAO), t -> log
- .error("Extract data and save failure.", t)), 5,
moduleConfig.getPersistentPeriod(),
- TimeUnit.SECONDS
- );
+ .scheduleWithFixedDelay(
+ new RunnableWithExceptionProtection(() ->
extractDataAndSave(batchDAO), t -> log
+ .error("Extract data and save failure.",
t)), 5, moduleConfig.getPersistentPeriod(),
+ TimeUnit.SECONDS
+ );
this.isStarted = true;
}
}
- private void extractDataAndSave(IBatchDAO batchDAO) {
+ @VisibleForTesting
+ void extractDataAndSave(IBatchDAO batchDAO) {
if (log.isDebugEnabled()) {
log.debug("Extract data and save");
}
long startTime = System.currentTimeMillis();
+ HistogramMetrics.Timer allTimer = allLatency.createTimer();
try {
- HistogramMetrics.Timer timer = prepareLatency.createTimer();
+ List<PersistenceWorker> persistenceWorkers = new ArrayList<>();
Review comment:
```suggestion
List<PersistenceWorker<? extends StorageData>>
persistenceWorkers = new ArrayList<>();
```
##########
File path:
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/PersistenceTimer.java
##########
@@ -63,98 +70,150 @@ public void start(ModuleManager moduleManager,
CoreModuleConfig moduleConfig) {
IBatchDAO batchDAO =
moduleManager.find(StorageModule.NAME).provider().getService(IBatchDAO.class);
MetricsCreator metricsCreator =
moduleManager.find(TelemetryModule.NAME)
- .provider()
-
.getService(MetricsCreator.class);
+ .provider()
+ .getService(MetricsCreator.class);
errorCounter = metricsCreator.createCounter(
- "persistence_timer_bulk_error_count", "Error execution of the
prepare stage in persistence timer",
- MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ "persistence_timer_bulk_error_count", "Error execution of the
prepare stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
);
prepareLatency = metricsCreator.createHistogramMetric(
- "persistence_timer_bulk_prepare_latency", "Latency of the prepare
stage in persistence timer",
- MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ "persistence_timer_bulk_prepare_latency", "Latency of the
prepare stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
);
executeLatency = metricsCreator.createHistogramMetric(
- "persistence_timer_bulk_execute_latency", "Latency of the execute
stage in persistence timer",
- MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ "persistence_timer_bulk_execute_latency", "Latency of the
execute stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
+ );
+ allLatency = metricsCreator.createHistogramMetric(
+ "persistence_timer_bulk_all_latency", "Latency of the all
stage in persistence timer",
+ MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE
);
+
syncOperationThreadsNum = moduleConfig.getSyncThreads();
maxSyncoperationNum = moduleConfig.getMaxSyncOperationNum();
+ batchExecutorService = Executors.newSingleThreadExecutor();
executorService =
Executors.newFixedThreadPool(syncOperationThreadsNum);
+ prepareExecutorService =
Executors.newFixedThreadPool(moduleConfig.getPrepareThreads());
if (!isStarted) {
Executors.newSingleThreadScheduledExecutor()
- .scheduleWithFixedDelay(
- new RunnableWithExceptionProtection(() ->
extractDataAndSave(batchDAO), t -> log
- .error("Extract data and save failure.", t)), 5,
moduleConfig.getPersistentPeriod(),
- TimeUnit.SECONDS
- );
+ .scheduleWithFixedDelay(
+ new RunnableWithExceptionProtection(() ->
extractDataAndSave(batchDAO), t -> log
+ .error("Extract data and save failure.",
t)), 5, moduleConfig.getPersistentPeriod(),
+ TimeUnit.SECONDS
+ );
this.isStarted = true;
}
}
- private void extractDataAndSave(IBatchDAO batchDAO) {
+ @VisibleForTesting
+ void extractDataAndSave(IBatchDAO batchDAO) {
if (log.isDebugEnabled()) {
log.debug("Extract data and save");
}
long startTime = System.currentTimeMillis();
+ HistogramMetrics.Timer allTimer = allLatency.createTimer();
try {
- HistogramMetrics.Timer timer = prepareLatency.createTimer();
+ List<PersistenceWorker> persistenceWorkers = new ArrayList<>();
+
persistenceWorkers.addAll(TopNStreamProcessor.getInstance().getPersistentWorkers());
+
persistenceWorkers.addAll(MetricsStreamProcessor.getInstance().getPersistentWorkers());
+ CountDownLatch countDownLatch = new
CountDownLatch(MetricsStreamProcessor.getInstance().getPersistentWorkers().size());
Review comment:
I believe it should be
```suggestion
CountDownLatch countDownLatch = new
CountDownLatch(persistenceWorkers.size());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
