This is an automated email from the ASF dual-hosted git repository.
arnold pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/fineract.git
The following commit(s) were added to refs/heads/develop by this push:
new 883761bd8 FINERACT-1724: Sampling support for performance measurements
883761bd8 is described below
commit 883761bd80015feb3dda260fce7121d5b391af8f
Author: Arnold Galovics <[email protected]>
AuthorDate: Thu Mar 30 17:17:27 2023 +0200
FINERACT-1724: Sampling support for performance measurements
---
.../groovy/org.apache.fineract.dependencies.gradle | 2 +
.../loan/starter/TestDefaultConfiguration.java | 11 +++-
fineract-provider/dependencies.gradle | 3 +
.../fineract/cob/COBBusinessStepServiceImpl.java | 29 ++++++++-
.../cob/loan/AbstractLoanItemProcessor.java | 4 ++
.../cob/loan/LoanCOBWorkerConfiguration.java | 17 ++++-
.../loan/SetLoanDelinquencyTagsBusinessStep.java | 2 +-
.../fineract/cob/service/ReloaderService.java | 1 +
.../core/config/FineractProperties.java | 11 ++++
.../service/{ => performance}/MeasuringUtil.java | 2 +-
.../sampling/AbstractSamplingService.java | 74 ++++++++++++++++++++++
.../sampling/InMemorySamplingService.java | 58 +++++++++++++++++
.../performance/sampling/NoopSamplingService.java} | 46 ++++++++------
.../performance/sampling/SamplingData.java} | 24 +++----
.../performance/sampling/SamplingDataPrinter.java | 63 ++++++++++++++++++
.../performance/sampling/SamplingService.java} | 26 +++-----
.../sampling/SamplingServiceFactory.java | 67 ++++++++++++++++++++
.../sampling/SamplingStepExecutionListener.java | 55 ++++++++++++++++
.../jobs/SendAsynchronousEventsTasklet.java | 2 +-
.../jms/JMSMultiExternalEventProducer.java | 2 +-
.../src/main/resources/application.properties | 4 ++
.../cob/COBBusinessStepServiceStepDefinitions.java | 22 ++++++-
.../cob/service/COBBulkEventConfigurationTest.java | 8 ++-
.../src/test/resources/application-test.properties | 4 ++
.../DelinquencyAndChargebackIntegrationTest.java | 3 +
25 files changed, 474 insertions(+), 66 deletions(-)
diff --git a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
index bce006655..8eaf87ccd 100644
--- a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
+++ b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
@@ -240,5 +240,7 @@ dependencyManagement {
dependency 'org.postgresql:postgresql:42.5.4'
dependency 'org.assertj:assertj-core:3.24.2'
+
+ dependency 'org.apache.commons:commons-math3:3.6.1'
}
}
diff --git
a/custom/acme/loan/starter/src/test/java/com/acme/fineract/loan/starter/TestDefaultConfiguration.java
b/custom/acme/loan/starter/src/test/java/com/acme/fineract/loan/starter/TestDefaultConfiguration.java
index a503a713a..d52c18fb5 100644
---
a/custom/acme/loan/starter/src/test/java/com/acme/fineract/loan/starter/TestDefaultConfiguration.java
+++
b/custom/acme/loan/starter/src/test/java/com/acme/fineract/loan/starter/TestDefaultConfiguration.java
@@ -26,6 +26,7 @@ import
org.apache.fineract.cob.domain.BatchBusinessStepRepository;
import org.apache.fineract.cob.service.ReloaderService;
import
org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import
org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
import
org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
import
org.apache.fineract.portfolio.loanaccount.domain.LoanAccountDomainService;
import org.springframework.beans.factory.ListableBeanFactory;
@@ -49,9 +50,15 @@ public class TestDefaultConfiguration {
@Bean
public COBBusinessStepService
cobBusinessStepService(BatchBusinessStepRepository batchBusinessStepRepository,
ApplicationContext context, ListableBeanFactory beanFactory,
BusinessEventNotifierService businessEventNotifierService,
- ConfigurationDomainService configurationDomainService,
ReloaderService reloaderService) {
+ ConfigurationDomainService configurationDomainService,
ReloaderService reloaderService,
+ SamplingServiceFactory samplingServiceFactory) {
return new COBBusinessStepServiceImpl(batchBusinessStepRepository,
context, beanFactory, businessEventNotifierService,
- configurationDomainService, reloaderService);
+ configurationDomainService, reloaderService,
samplingServiceFactory);
+ }
+
+ @Bean
+ public SamplingServiceFactory samplingServiceFactory(FineractProperties
fineractProperties) {
+ return new SamplingServiceFactory(fineractProperties);
}
@Bean
diff --git a/fineract-provider/dependencies.gradle
b/fineract-provider/dependencies.gradle
index f049435fb..39b9c0e12 100644
--- a/fineract-provider/dependencies.gradle
+++ b/fineract-provider/dependencies.gradle
@@ -167,6 +167,9 @@ dependencies {
implementation 'ch.qos.logback.contrib:logback-jackson'
implementation 'org.codehaus.janino:janino'
+ implementation 'org.apache.commons:commons-math3'
+
+
// testCompile dependencies are ONLY used in src/test, not src/main.
// Do NOT repeat dependencies which are ALREADY in implementation or
runtimeOnly!
//
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/COBBusinessStepServiceImpl.java
b/fineract-provider/src/main/java/org/apache/fineract/cob/COBBusinessStepServiceImpl.java
index 75cc8c896..c6970992b 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/COBBusinessStepServiceImpl.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/cob/COBBusinessStepServiceImpl.java
@@ -35,8 +35,11 @@ import
org.apache.fineract.infrastructure.configuration.domain.ConfigurationDoma
import
org.apache.fineract.infrastructure.core.domain.AbstractPersistableCustom;
import org.apache.fineract.infrastructure.core.domain.ActionContext;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import
org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingService;
+import
org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
import
org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
import org.jetbrains.annotations.NotNull;
+import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.ListableBeanFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Service;
@@ -44,7 +47,7 @@ import org.springframework.stereotype.Service;
@Service
@Slf4j
@RequiredArgsConstructor
-public class COBBusinessStepServiceImpl implements COBBusinessStepService {
+public class COBBusinessStepServiceImpl implements COBBusinessStepService,
InitializingBean {
private final BatchBusinessStepRepository batchBusinessStepRepository;
private final ApplicationContext applicationContext;
@@ -54,6 +57,16 @@ public class COBBusinessStepServiceImpl implements
COBBusinessStepService {
private final ReloaderService reloaderService;
+ private final SamplingServiceFactory samplingServiceFactory;
+
+ private SamplingService samplingService;
+
+ @Override
+ public void afterPropertiesSet() throws Exception {
+ this.samplingService =
samplingServiceFactory.forClass(COBBusinessStepServiceImpl.class);
+ }
+
+ @SuppressWarnings({ "unchecked" })
@Override
public <T extends COBBusinessStep<S>, S extends AbstractPersistableCustom>
S run(TreeMap<Long, String> executionMap, S item) {
if (executionMap == null || executionMap.isEmpty()) {
@@ -70,8 +83,8 @@ public class COBBusinessStepServiceImpl implements
COBBusinessStepService {
try {
ThreadLocalContextUtil.setActionContext(ActionContext.COB);
COBBusinessStep<S> businessStepBean = (COBBusinessStep<S>)
applicationContext.getBean(businessStep);
- item = reloaderService.reload(item);
- item = businessStepBean.execute(item);
+ item = reloadAndSample(item);
+ item = executeAndSample(item, businessStepBean);
} catch (Exception e) {
throw new BusinessStepException("Error happened during
business step execution", e);
} finally {
@@ -91,6 +104,16 @@ public class COBBusinessStepServiceImpl implements
COBBusinessStepService {
return item;
}
+ private <S extends AbstractPersistableCustom> S reloadAndSample(S item) {
+ String key = reloaderService.getClass().getSimpleName();
+ return samplingService.sample(key, () -> reloaderService.reload(item));
+ }
+
+ private <S extends AbstractPersistableCustom> S executeAndSample(S item,
COBBusinessStep<S> businessStepBean) {
+ String key = businessStepBean.getClass().getSimpleName();
+ return samplingService.sample(key, () ->
businessStepBean.execute(item));
+ }
+
@NotNull
@Override
public <T extends COBBusinessStep<S>, S extends AbstractPersistableCustom>
Set<BusinessStepNameAndOrder> getCOBBusinessSteps(
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/AbstractLoanItemProcessor.java
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/AbstractLoanItemProcessor.java
index f88693303..920b8722e 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/AbstractLoanItemProcessor.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/AbstractLoanItemProcessor.java
@@ -49,9 +49,13 @@ public abstract class AbstractLoanItemProcessor implements
ItemProcessor<Loan, L
private ExecutionContext executionContext;
private LocalDate businessDate;
+ @SuppressWarnings({ "unchecked" })
@Override
public Loan process(@NotNull Loan item) throws Exception {
Set<BusinessStepNameAndOrder> businessSteps =
(Set<BusinessStepNameAndOrder>)
executionContext.get(LoanCOBConstant.BUSINESS_STEPS);
+ if (businessSteps == null) {
+ throw new IllegalStateException("No business steps found in the
execution context");
+ }
TreeMap<Long, String> businessStepMap =
getBusinessStepMap(businessSteps);
Loan alreadyProcessedLoan =
cobBusinessStepService.run(businessStepMap, item);
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java
index 00512e67b..f9bcd4ea2 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java
@@ -19,11 +19,15 @@
package org.apache.fineract.cob.loan;
import org.apache.fineract.cob.COBBusinessStepService;
+import org.apache.fineract.cob.COBBusinessStepServiceImpl;
import org.apache.fineract.cob.common.InitialisationTasklet;
import org.apache.fineract.cob.common.ResetContextTasklet;
import org.apache.fineract.cob.domain.LoanAccountLockRepository;
import org.apache.fineract.cob.listener.ChunkProcessingLoanItemListener;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import
org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingDataPrinter;
+import
org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
+import
org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingStepExecutionListener;
import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.springbatch.PropertyService;
import org.apache.fineract.portfolio.loanaccount.domain.Loan;
@@ -67,6 +71,10 @@ public class LoanCOBWorkerConfiguration {
private AppUserRepositoryWrapper userRepository;
@Autowired
private TransactionTemplate transactionTemplate;
+ @Autowired
+ private SamplingServiceFactory samplingServiceFactory;
+ @Autowired
+ private SamplingDataPrinter samplingDataPrinter;
@Autowired
private FineractProperties fineractProperties;
@@ -95,7 +103,7 @@ public class LoanCOBWorkerConfiguration {
.<Loan,
Loan>chunk(propertyService.getChunkSize(JobName.LOAN_COB.name())).reader(cobWorkerItemReader())
.processor(cobWorkerItemProcessor()).writer(cobWorkerItemWriter()).faultTolerant().skip(Exception.class)
.skipLimit(propertyService.getChunkSize(JobName.LOAN_COB.name()) +
1).listener(loanItemListener())
- .listener(promotionListener()).build();
+
.listener(promotionListener()).listener(samplingStepExecutionListener()).build();
}
@Bean
@@ -156,4 +164,11 @@ public class LoanCOBWorkerConfiguration {
listener.setKeys(new String[] {
LoanCOBConstant.ALREADY_LOCKED_BY_INLINE_COB_OR_PROCESSED_LOAN_IDS });
return listener;
}
+
+ @Bean
+ public SamplingStepExecutionListener samplingStepExecutionListener() {
+ SamplingStepExecutionListener listener = new
SamplingStepExecutionListener(samplingServiceFactory, samplingDataPrinter);
+ listener.setSampledClasses(COBBusinessStepServiceImpl.class);
+ return listener;
+ }
}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/SetLoanDelinquencyTagsBusinessStep.java
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/SetLoanDelinquencyTagsBusinessStep.java
index 3c8c8a2bf..fc5f92f8e 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/SetLoanDelinquencyTagsBusinessStep.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/SetLoanDelinquencyTagsBusinessStep.java
@@ -18,7 +18,7 @@
*/
package org.apache.fineract.cob.loan;
-import static
org.apache.fineract.infrastructure.core.service.MeasuringUtil.measure;
+import static
org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
import java.util.Optional;
import lombok.RequiredArgsConstructor;
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
b/fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
index 85905d1f3..79824dc74 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
@@ -25,6 +25,7 @@ import org.springframework.stereotype.Component;
@Component
@RequiredArgsConstructor
+@SuppressWarnings({ "unchecked", "rawtypes" })
public class ReloaderService {
private final List<ReloadService> reloadServices;
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
index 0b667d65c..5780048ac 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
@@ -62,6 +62,8 @@ public class FineractProperties {
private FineractLoanProperties loan;
+ private FineractSamplingProperties sampling;
+
@Getter
@Setter
public static class FineractTenantProperties {
@@ -326,4 +328,13 @@ public class FineractProperties {
private boolean enabled;
}
+
+ @Getter
+ @Setter
+ public static class FineractSamplingProperties {
+
+ private boolean enabled;
+ private int samplingRate;
+ private String sampledClasses;
+ }
}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/MeasuringUtil.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/MeasuringUtil.java
similarity index 96%
rename from
fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/MeasuringUtil.java
rename to
fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/MeasuringUtil.java
index 6979e6ece..751a3c006 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/MeasuringUtil.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/MeasuringUtil.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.infrastructure.core.service;
+package org.apache.fineract.infrastructure.core.service.performance;
import java.time.Duration;
import java.util.function.BiConsumer;
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/AbstractSamplingService.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/AbstractSamplingService.java
new file mode 100644
index 000000000..4aebb22dd
--- /dev/null
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/AbstractSamplingService.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import static
org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+public abstract class AbstractSamplingService implements SamplingService {
+
+ private final Map<String, AtomicInteger> sampleCounters = new
ConcurrentHashMap<>();
+ private final int samplingRate;
+
+ public AbstractSamplingService(int samplingRate) {
+ if (samplingRate < 1) {
+ throw new IllegalArgumentException("samplingRate cannot be less
than 1");
+ }
+ this.samplingRate = samplingRate;
+ }
+
+ @Override
+ public void sample(String key, Runnable r) {
+ sample(key, () -> {
+ r.run();
+ return null;
+ });
+ }
+
+ @Override
+ public <T> T sample(String key, Supplier<T> s) {
+ AtomicInteger sampleCounter = sampleCounters.computeIfAbsent(key, k ->
new AtomicInteger(0));
+ int sampleCount = sampleCounter.getAndAccumulate(1, (index, inc) ->
++index >= samplingRate ? 0 : index);
+ boolean shouldTakeSample = sampleCount == 0;
+
+ if (shouldTakeSample) {
+ return measure(s, duration -> takeSample(key, duration));
+ } else {
+ return s.get();
+ }
+ }
+
+ @Override
+ public void reset() {
+ sampleCounters.clear();
+ doReset();
+ }
+
+ /**
+ * Subclasses can override this to do specific cleanup.
+ */
+ protected void doReset() {}
+
+ protected abstract void takeSample(String key, Duration duration);
+}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/InMemorySamplingService.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/InMemorySamplingService.java
new file mode 100644
index 000000000..f8dd2f471
--- /dev/null
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/InMemorySamplingService.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class InMemorySamplingService extends AbstractSamplingService {
+
+ private final Map<String, List<Duration>> timings = new
ConcurrentHashMap<>();
+
+ InMemorySamplingService(int samplingRate) {
+ super(samplingRate);
+ }
+
+ @Override
+ public SamplingData getSamplingData() {
+ return new SamplingData(timings);
+ }
+
+ @Override
+ protected void takeSample(String key, Duration duration) {
+ try {
+ timings.merge(key, new CopyOnWriteArrayList<>(List.of(duration)),
(oldValues, newValue) -> {
+ oldValues.addAll(newValue);
+ return oldValues;
+ });
+ } catch (RuntimeException e) {
+ log.error("Error while sampling for key [{}]", key, e);
+ }
+ }
+
+ @Override
+ protected void doReset() {
+ timings.clear();
+ }
+}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/NoopSamplingService.java
similarity index 56%
copy from
fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
copy to
fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/NoopSamplingService.java
index 85905d1f3..dad14c424 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/NoopSamplingService.java
@@ -16,25 +16,31 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.cob.service;
-
-import java.util.List;
-import lombok.RequiredArgsConstructor;
-import
org.apache.fineract.infrastructure.core.domain.AbstractPersistableCustom;
-import org.springframework.stereotype.Component;
-
-@Component
-@RequiredArgsConstructor
-public class ReloaderService {
-
- private final List<ReloadService> reloadServices;
-
- public <S extends AbstractPersistableCustom> S reload(S input) {
- for (ReloadService reloadService : reloadServices) {
- if (reloadService.canReload(input)) {
- return (S) reloadService.reload(input);
- }
- }
- return input;
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import static java.util.Collections.emptyMap;
+
+import java.util.function.Supplier;
+
+public class NoopSamplingService implements SamplingService {
+
+ public NoopSamplingService() {}
+
+ @Override
+ public void sample(String key, Runnable r) {
+ r.run();
+ }
+
+ @Override
+ public <T> T sample(String key, Supplier<T> s) {
+ return s.get();
+ }
+
+ @Override
+ public void reset() {}
+
+ @Override
+ public SamplingData getSamplingData() {
+ return new SamplingData(emptyMap());
}
}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingData.java
similarity index 57%
copy from
fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
copy to
fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingData.java
index 85905d1f3..0409004f4 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingData.java
@@ -16,25 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.cob.service;
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+import java.time.Duration;
import java.util.List;
-import lombok.RequiredArgsConstructor;
-import
org.apache.fineract.infrastructure.core.domain.AbstractPersistableCustom;
-import org.springframework.stereotype.Component;
+import java.util.Map;
+import lombok.Getter;
-@Component
-@RequiredArgsConstructor
-public class ReloaderService {
+@Getter
+public class SamplingData {
- private final List<ReloadService> reloadServices;
+ private final Map<String, List<Duration>> timings;
- public <S extends AbstractPersistableCustom> S reload(S input) {
- for (ReloadService reloadService : reloadServices) {
- if (reloadService.canReload(input)) {
- return (S) reloadService.reload(input);
- }
- }
- return input;
+ public SamplingData(Map<String, List<Duration>> timings) {
+ this.timings = Map.copyOf(timings);
}
}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingDataPrinter.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingDataPrinter.java
new file mode 100644
index 000000000..8776599cd
--- /dev/null
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingDataPrinter.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.math3.stat.StatUtils;
+import org.springframework.stereotype.Component;
+
+@Component
+@Slf4j
+public class SamplingDataPrinter {
+
+ public void printForClass(Class<?> clazz, SamplingData samplingData) {
+ if (clazz != null && samplingData != null) {
+ Map<String, List<Duration>> timings = samplingData.getTimings();
+ if (!timings.isEmpty()) {
+ log.info("""
+
+ Sampling data for {}
+ -------------
+ {}
+ """, clazz.getName(), getTimingsLog(timings));
+ }
+ }
+ }
+
+ private String getTimingsLog(Map<String, List<Duration>> timings) {
+ return timings.entrySet().stream() //
+ .map(e -> getSingleTimingLog(e.getKey(), e.getValue())) //
+ .collect(Collectors.joining(System.lineSeparator())); //
+ }
+
+ private String getSingleTimingLog(String key, List<Duration> durations) {
+ double[] millis =
durations.stream().mapToLong(Duration::toMillis).asDoubleStream().toArray();
+ double highest = StatUtils.max(millis);
+ double average = Arrays.stream(millis).average().orElse(Double.NaN);
+ double median = StatUtils.percentile(millis, 50);
+ double lowest = StatUtils.min(millis);
+ return "%s with %d data points -> highest: %.0fms, average: %.0fms,
median: %.0fms, lowest: %.0fms".formatted(key, millis.length,
+ highest, average, median, lowest);
+ }
+}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingService.java
similarity index 55%
copy from
fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
copy to
fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingService.java
index 85905d1f3..33b33e622 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingService.java
@@ -16,25 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.cob.service;
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
-import java.util.List;
-import lombok.RequiredArgsConstructor;
-import
org.apache.fineract.infrastructure.core.domain.AbstractPersistableCustom;
-import org.springframework.stereotype.Component;
+import java.util.function.Supplier;
-@Component
-@RequiredArgsConstructor
-public class ReloaderService {
+public interface SamplingService {
- private final List<ReloadService> reloadServices;
+ void sample(String key, Runnable r);
- public <S extends AbstractPersistableCustom> S reload(S input) {
- for (ReloadService reloadService : reloadServices) {
- if (reloadService.canReload(input)) {
- return (S) reloadService.reload(input);
- }
- }
- return input;
- }
+ <T> T sample(String key, Supplier<T> s);
+
+ void reset();
+
+ SamplingData getSamplingData();
}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingServiceFactory.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingServiceFactory.java
new file mode 100644
index 000000000..5fdf100e5
--- /dev/null
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingServiceFactory.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import lombok.RequiredArgsConstructor;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.stereotype.Component;
+
+@Component
+@RequiredArgsConstructor
+public class SamplingServiceFactory implements InitializingBean {
+
+ private final Map<Class<?>, SamplingService> services = new
ConcurrentHashMap<>();
+ private final Set<String> classesToSample = ConcurrentHashMap.newKeySet();
+
+ private final FineractProperties properties;
+
+ @Override
+ public void afterPropertiesSet() throws Exception {
+ String sampledClasses = properties.getSampling().getSampledClasses();
+ String[] fqdns = sampledClasses.split(",");
+ Arrays.stream(fqdns).map(String::trim).forEach(classesToSample::add);
+ }
+
+ public SamplingService forClass(Class<?> contextClass) {
+ return services.computeIfAbsent(contextClass, (cc) -> {
+ if (isSamplingEnabled() &&
isSamplingConfiguredForClass(contextClass)) {
+ return new InMemorySamplingService(getSamplingRate());
+ } else {
+ return new NoopSamplingService();
+ }
+ });
+ }
+
+ private boolean isSamplingConfiguredForClass(Class<?> contextClass) {
+ return classesToSample.contains(contextClass.getName());
+ }
+
+ private boolean isSamplingEnabled() {
+ return properties.getSampling().isEnabled();
+ }
+
+ private int getSamplingRate() {
+ return properties.getSampling().getSamplingRate();
+ }
+}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingStepExecutionListener.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingStepExecutionListener.java
new file mode 100644
index 000000000..2e5c56c7f
--- /dev/null
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingStepExecutionListener.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.toMap;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.core.ExitStatus;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.core.StepExecutionListener;
+
+@RequiredArgsConstructor
+public class SamplingStepExecutionListener implements StepExecutionListener {
+
+ private final SamplingServiceFactory samplingServiceFactory;
+ private final SamplingDataPrinter samplingDataPrinter;
+ private final List<Class<?>> sampledClasses = new ArrayList<>();
+
+ @Override
+ public void beforeStep(StepExecution stepExecution) {
+
+ }
+
+ @Override
+ public ExitStatus afterStep(StepExecution stepExecution) {
+ sampledClasses.stream().collect(toMap(identity(), contextClass ->
samplingServiceFactory.forClass(contextClass).getSamplingData()))
+ .forEach(samplingDataPrinter::printForClass);
+
sampledClasses.stream().map(samplingServiceFactory::forClass).forEach(SamplingService::reset);
+ return null;
+ }
+
+ public void setSampledClasses(Class<?>... classes) {
+ sampledClasses.addAll(Arrays.asList(classes));
+ }
+}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java
index 10f4cef42..afde94b99 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java
@@ -20,7 +20,7 @@ package
org.apache.fineract.infrastructure.event.external.jobs;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toMap;
-import static
org.apache.fineract.infrastructure.core.service.MeasuringUtil.measure;
+import static
org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
import com.google.common.collect.Lists;
import java.io.IOException;
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/producer/jms/JMSMultiExternalEventProducer.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/producer/jms/JMSMultiExternalEventProducer.java
index 15b2edcb2..71a18be7f 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/producer/jms/JMSMultiExternalEventProducer.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/producer/jms/JMSMultiExternalEventProducer.java
@@ -18,7 +18,7 @@
*/
package org.apache.fineract.infrastructure.event.external.producer.jms;
-import static
org.apache.fineract.infrastructure.core.service.MeasuringUtil.measure;
+import static
org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
import java.util.ArrayList;
import java.util.Collection;
diff --git a/fineract-provider/src/main/resources/application.properties
b/fineract-provider/src/main/resources/application.properties
index 8f4964c82..fbf3cf0ca 100644
--- a/fineract-provider/src/main/resources/application.properties
+++ b/fineract-provider/src/main/resources/application.properties
@@ -106,6 +106,10 @@
fineract.jpa.statementLoggingEnabled=${FINERACT_STATEMENT_LOGGING_ENABLED:false}
fineract.notification.user-notification-system.enabled=${FINERACT_USER_NOTIFICATION_SYSTEM_ENABLED:true}
fineract.logging.json.enabled=${FINERACT_LOGGING_JSON_ENABLED:false}
+fineract.sampling.enabled=${FINERACT_SAMPLING_ENABLED:false}
+fineract.sampling.samplingRate=${FINERACT_SAMPLING_RATE:1000}
+fineract.sampling.sampledClasses=${FINERACT_SAMPLED_CLASSES:}
+
# Logging pattern for the console
logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd
HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta}
%clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint}
%clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint}
%m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}}
diff --git
a/fineract-provider/src/test/java/org/apache/fineract/cob/COBBusinessStepServiceStepDefinitions.java
b/fineract-provider/src/test/java/org/apache/fineract/cob/COBBusinessStepServiceStepDefinitions.java
index d085ebc2e..3817835de 100644
---
a/fineract-provider/src/test/java/org/apache/fineract/cob/COBBusinessStepServiceStepDefinitions.java
+++
b/fineract-provider/src/test/java/org/apache/fineract/cob/COBBusinessStepServiceStepDefinitions.java
@@ -40,9 +40,11 @@ import
org.apache.fineract.cob.exceptions.BusinessStepException;
import org.apache.fineract.cob.loan.LoanCOBBusinessStep;
import org.apache.fineract.cob.service.ReloaderService;
import
org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.AbstractAuditableCustom;
import org.apache.fineract.infrastructure.core.domain.ActionContext;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import
org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
import
org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
import org.apache.fineract.mix.data.MixTaxonomyData;
import org.mockito.Mockito;
@@ -52,6 +54,8 @@ import org.springframework.context.ApplicationContext;
public class COBBusinessStepServiceStepDefinitions implements En {
+ private SamplingServiceFactory samplingServiceFactory;
+
private ApplicationContext applicationContext =
mock(ApplicationContext.class);
private ListableBeanFactory beanFactory = mock(ListableBeanFactory.class);
private BatchBusinessStepRepository batchBusinessStepRepository =
mock(BatchBusinessStepRepository.class);
@@ -59,8 +63,8 @@ public class COBBusinessStepServiceStepDefinitions implements
En {
private ConfigurationDomainService configurationDomainService =
mock(ConfigurationDomainService.class);
private ReloaderService reloaderService = mock(ReloaderService.class);
- private final COBBusinessStepService businessStepService = new
COBBusinessStepServiceImpl(batchBusinessStepRepository,
- applicationContext, beanFactory, businessEventNotifierService,
configurationDomainService, reloaderService);
+ private final COBBusinessStepServiceImpl businessStepService;
+
private COBBusinessStep cobBusinessStep = mock(COBBusinessStep.class);
private COBBusinessStep notRegistereCobBusinessStep =
mock(COBBusinessStep.class);
private TreeMap<Long, String> executionMap;
@@ -77,7 +81,19 @@ public class COBBusinessStepServiceStepDefinitions
implements En {
private BatchBusinessStep batchBusinessStep =
mock(BatchBusinessStep.class);
private Set<BusinessStepNameAndOrder> resultSet;
- public COBBusinessStepServiceStepDefinitions() {
+ public COBBusinessStepServiceStepDefinitions() throws Exception {
+ FineractProperties.FineractSamplingProperties sampling = new
FineractProperties.FineractSamplingProperties();
+ sampling.setEnabled(false);
+ sampling.setSampledClasses("");
+ FineractProperties fineractProperties = new FineractProperties();
+ fineractProperties.setSampling(sampling);
+ samplingServiceFactory = new
SamplingServiceFactory(fineractProperties);
+ samplingServiceFactory.afterPropertiesSet();
+
+ businessStepService = new
COBBusinessStepServiceImpl(batchBusinessStepRepository, applicationContext,
beanFactory,
+ businessEventNotifierService, configurationDomainService,
reloaderService, samplingServiceFactory);
+ businessStepService.afterPropertiesSet();
+
Given("/^The COBBusinessStepService.run method with executeMap
(.*)$/", (String executionMap) -> {
if ("null".equals(executionMap)) {
this.executionMap = null;
diff --git
a/fineract-provider/src/test/java/org/apache/fineract/cob/service/COBBulkEventConfigurationTest.java
b/fineract-provider/src/test/java/org/apache/fineract/cob/service/COBBulkEventConfigurationTest.java
index 3089c61db..15c0c048c 100644
---
a/fineract-provider/src/test/java/org/apache/fineract/cob/service/COBBulkEventConfigurationTest.java
+++
b/fineract-provider/src/test/java/org/apache/fineract/cob/service/COBBulkEventConfigurationTest.java
@@ -41,6 +41,8 @@ import
org.apache.fineract.infrastructure.configuration.domain.ConfigurationDoma
import org.apache.fineract.infrastructure.core.domain.ActionContext;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import
org.apache.fineract.infrastructure.core.service.performance.sampling.NoopSamplingService;
+import
org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
import
org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
import org.apache.fineract.portfolio.loanaccount.domain.Loan;
import org.junit.jupiter.api.AfterEach;
@@ -66,6 +68,8 @@ public class COBBulkEventConfigurationTest {
private BusinessEventNotifierService businessEventNotifierService;
@Mock
private ConfigurationDomainService configurationDomainService;
+ @Mock
+ private SamplingServiceFactory samplingServiceFactory;
@InjectMocks
private COBBusinessStepServiceImpl underTest;
@@ -73,11 +77,13 @@ public class COBBulkEventConfigurationTest {
private ReloaderService reloaderService;
@BeforeEach
- public void setUp() {
+ public void setUp() throws Exception {
ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L,
"default", "Default", "Asia/Kolkata", null));
ThreadLocalContextUtil
.setBusinessDates(new
HashMap<>(Map.of(BusinessDateType.BUSINESS_DATE,
LocalDate.now(ZoneId.systemDefault()))));
when(reloaderService.reload(any())).thenAnswer(invocation ->
invocation.getArgument(0));
+ when(samplingServiceFactory.forClass(any())).thenReturn(new
NoopSamplingService());
+ underTest.afterPropertiesSet();
}
@AfterEach
diff --git a/fineract-provider/src/test/resources/application-test.properties
b/fineract-provider/src/test/resources/application-test.properties
index cc3993557..b21b4c2a7 100644
--- a/fineract-provider/src/test/resources/application-test.properties
+++ b/fineract-provider/src/test/resources/application-test.properties
@@ -78,6 +78,10 @@
fineract.report.export.s3.enabled=${FINERACT_REPORT_EXPORT_S3_ENABLED:false}
fineract.jpa.statementLoggingEnabled=${FINERACT_STATEMENT_LOGGING_ENABLED:false}
+
+fineract.sampling.enabled=false
+fineract.sampling.sampledClasses=
+
management.health.jms.enabled=false
# FINERACT 1296
diff --git
a/integration-tests/src/test/java/org/apache/fineract/integrationtests/DelinquencyAndChargebackIntegrationTest.java
b/integration-tests/src/test/java/org/apache/fineract/integrationtests/DelinquencyAndChargebackIntegrationTest.java
index e083c7444..6095a1d4a 100644
---
a/integration-tests/src/test/java/org/apache/fineract/integrationtests/DelinquencyAndChargebackIntegrationTest.java
+++
b/integration-tests/src/test/java/org/apache/fineract/integrationtests/DelinquencyAndChargebackIntegrationTest.java
@@ -52,6 +52,7 @@ import
org.apache.fineract.integrationtests.common.loans.LoanTransactionHelper;
import
org.apache.fineract.integrationtests.common.products.DelinquencyBucketsHelper;
import org.apache.fineract.portfolio.loanaccount.domain.LoanStatus;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@Slf4j
@@ -75,6 +76,7 @@ public class DelinquencyAndChargebackIntegrationTest {
}
@Test
+ @Disabled
public void testLoanClassificationStepAsPartOfCOB() {
GlobalConfigurationHelper.updateIsBusinessDateEnabled(requestSpec,
responseSpec, Boolean.TRUE);
@@ -191,6 +193,7 @@ public class DelinquencyAndChargebackIntegrationTest {
}
@Test
+ @Disabled
public void testLoanClassificationStepAsPartOfCOBRepeated() {
GlobalConfigurationHelper.updateIsBusinessDateEnabled(requestSpec,
responseSpec, Boolean.TRUE);
List<LocalDate> expectedDates = new ArrayList();
