This is an automated email from the ASF dual-hosted git repository.
aleks pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/fineract.git
The following commit(s) were added to refs/heads/develop by this push:
new 071cdae69 [FINERACT-1724] making partition for big parameter lists
071cdae69 is described below
commit 071cdae69bf67e64e28b54dc866167335add182e
Author: taskain7 <[email protected]>
AuthorDate: Mon Mar 13 08:40:04 2023 +0100
[FINERACT-1724] making partition for big parameter lists
---
.../fineract/cob/loan/ApplyLoanLockTasklet.java | 8 ++++--
.../fineract/cob/loan/FetchAndLockLoanTasklet.java | 10 ++++++-
.../cob/loan/LoanCOBManagerConfiguration.java | 5 +++-
.../cob/loan/LoanCOBWorkerConfiguration.java | 6 +++-
.../service/InlineLoanCOBExecutorServiceImpl.java | 9 +++++-
.../core/config/FineractProperties.java | 10 +++++++
.../jobs/filter/LoanCOBApiFilter.java | 12 ++++++--
.../AccountTransfersWritePlatformServiceImpl.java | 12 ++++++--
.../loanaccount/domain/LoanRepository.java | 4 ---
.../loanaccount/domain/LoanRepositoryWrapper.java | 18 ++++++------
.../src/main/resources/application.properties | 2 ++
.../loan/ApplyLoanLockTaskletStepDefinitions.java | 9 +++++-
.../cob/loan/FetchAndLockLoanStepDefinitions.java | 14 +++++++++-
.../InlineLoanCOBExecutorServiceImplTest.java | 32 ++++++++++++++++++++++
.../jobs/filter/LoanCOBApiFilterTest.java | 31 ++++++++++++++++-----
15 files changed, 150 insertions(+), 32 deletions(-)
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java
index fbfb39a47..35183807c 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java
@@ -18,6 +18,7 @@
*/
package org.apache.fineract.cob.loan;
+import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -28,6 +29,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.cob.domain.LoanAccountLock;
import org.apache.fineract.cob.domain.LoanAccountLockRepository;
import org.apache.fineract.cob.domain.LockOwner;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.jetbrains.annotations.NotNull;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
@@ -40,13 +42,15 @@ import org.springframework.batch.repeat.RepeatStatus;
public class ApplyLoanLockTasklet implements Tasklet {
private final LoanAccountLockRepository accountLockRepository;
+ private final FineractProperties fineractProperties;
@Override
public RepeatStatus execute(@NotNull StepContribution contribution,
@NotNull ChunkContext chunkContext) throws Exception {
ExecutionContext executionContext =
contribution.getStepExecution().getExecutionContext();
List<Long> loanIds = (List<Long>)
executionContext.get(LoanCOBConstant.LOAN_IDS);
-
- List<LoanAccountLock> accountLocks =
accountLockRepository.findAllByLoanIdIn(loanIds);
+ List<List<Long>> loanIdPartitions = Lists.partition(loanIds,
fineractProperties.getQuery().getInClauseParameterSizeLimit());
+ List<LoanAccountLock> accountLocks = new ArrayList<>();
+ loanIdPartitions.forEach(loanIdPartition ->
accountLocks.addAll(accountLockRepository.findAllByLoanIdIn(loanIdPartition)));
Map<Long, LoanAccountLock> alreadySoftLockedAccountsMap =
accountLocks.stream()
.filter(e ->
LockOwner.LOAN_COB_PARTITIONING.equals(e.getLockOwner()))
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/FetchAndLockLoanTasklet.java
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/FetchAndLockLoanTasklet.java
index c864ba396..f11c8576b 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/FetchAndLockLoanTasklet.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/FetchAndLockLoanTasklet.java
@@ -18,6 +18,7 @@
*/
package org.apache.fineract.cob.loan;
+import com.google.common.collect.Lists;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
@@ -27,6 +28,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.cob.domain.LoanAccountLock;
import org.apache.fineract.cob.domain.LoanAccountLockRepository;
import org.apache.fineract.cob.domain.LockOwner;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.jetbrains.annotations.NotNull;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
@@ -43,6 +45,8 @@ public class FetchAndLockLoanTasklet implements Tasklet {
private final RetrieveLoanIdService retrieveLoanIdService;
+ private final FineractProperties fineractProperties;
+
@Override
public RepeatStatus execute(@NotNull StepContribution contribution,
@NotNull ChunkContext chunkContext) throws Exception {
String businessDateParameter = (String)
contribution.getStepExecution().getJobExecution().getExecutionContext()
@@ -54,7 +58,11 @@ public class FetchAndLockLoanTasklet implements Tasklet {
}
List<Long> remainingIds = new ArrayList<>(allNonClosedLoanIds);
- List<LoanAccountLock> loanAccountLocks =
loanAccountLockRepository.findAllByLoanIdIn(remainingIds);
+ List<List<Long>> remainingIdPartitions = Lists.partition(remainingIds,
+ fineractProperties.getQuery().getInClauseParameterSizeLimit());
+ List<LoanAccountLock> loanAccountLocks = new ArrayList<>();
+ remainingIdPartitions.forEach(
+ remainingIdPartition ->
loanAccountLocks.addAll(loanAccountLockRepository.findAllByLoanIdIn(remainingIdPartition)));
List<Long> alreadySoftLockedAccounts = loanAccountLocks.stream()
.filter(e ->
LockOwner.LOAN_COB_PARTITIONING.equals(e.getLockOwner())).map(LoanAccountLock::getLoanId).toList();
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java
index 4fa8ca2e3..eb8b7695f 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java
@@ -23,6 +23,7 @@ import org.apache.fineract.cob.COBBusinessStepService;
import org.apache.fineract.cob.common.CustomJobParameterResolver;
import org.apache.fineract.cob.domain.LoanAccountLockRepository;
import org.apache.fineract.cob.listener.COBExecutionListenerRunner;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import
org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.springbatch.PropertyService;
@@ -79,6 +80,8 @@ public class LoanCOBManagerConfiguration {
private CustomJobParameterResolver customJobParameterResolver;
@Autowired
private LoanRepository loanRepository;
+ @Autowired
+ private FineractProperties fineractProperties;
@Bean
@JobScope
@@ -111,7 +114,7 @@ public class LoanCOBManagerConfiguration {
@Bean
@JobScope
public FetchAndLockLoanTasklet fetchAndLockLoanTasklet() {
- return new FetchAndLockLoanTasklet(accountLockRepository,
retrieveLoanIdService);
+ return new FetchAndLockLoanTasklet(accountLockRepository,
retrieveLoanIdService, fineractProperties);
}
@Bean
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java
index 0f6bccbb7..00512e67b 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java
@@ -23,6 +23,7 @@ import org.apache.fineract.cob.common.InitialisationTasklet;
import org.apache.fineract.cob.common.ResetContextTasklet;
import org.apache.fineract.cob.domain.LoanAccountLockRepository;
import org.apache.fineract.cob.listener.ChunkProcessingLoanItemListener;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.springbatch.PropertyService;
import org.apache.fineract.portfolio.loanaccount.domain.Loan;
@@ -67,6 +68,9 @@ public class LoanCOBWorkerConfiguration {
@Autowired
private TransactionTemplate transactionTemplate;
+ @Autowired
+ private FineractProperties fineractProperties;
+
@Bean(name = LoanCOBConstant.LOAN_COB_WORKER_STEP)
public Step loanCOBWorkerStep() {
return stepBuilderFactory.get("Loan COB worker -
Step").inputChannel(inboundRequests).flow(flow()).build();
@@ -118,7 +122,7 @@ public class LoanCOBWorkerConfiguration {
@Bean
public ApplyLoanLockTasklet applyLock() {
- return new ApplyLoanLockTasklet(accountLockRepository);
+ return new ApplyLoanLockTasklet(accountLockRepository,
fineractProperties);
}
@Bean
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImpl.java
b/fineract-provider/src/main/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImpl.java
index 12e31614e..838f06cb3 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImpl.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImpl.java
@@ -20,6 +20,7 @@ package org.apache.fineract.cob.service;
import static
org.springframework.transaction.TransactionDefinition.PROPAGATION_REQUIRES_NEW;
+import com.google.common.collect.Lists;
import com.google.gson.Gson;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
@@ -40,6 +41,7 @@ import
org.apache.fineract.cob.exceptions.LoanAccountLockCannotBeOverruledExcept
import org.apache.fineract.cob.loan.LoanCOBConstant;
import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
import org.apache.fineract.infrastructure.core.api.JsonCommand;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
import
org.apache.fineract.infrastructure.core.data.CommandProcessingResultBuilder;
import
org.apache.fineract.infrastructure.core.exception.PlatformInternalServerException;
@@ -85,6 +87,7 @@ public class InlineLoanCOBExecutorServiceImpl implements
InlineExecutorService<L
private final CustomJobParameterRepository customJobParameterRepository;
private final PlatformSecurityContext context;
private final LoanRepository loanRepository;
+ private final FineractProperties fineractProperties;
private final Gson gson = GoogleGsonSerializerHelper.createSimpleGson();
@@ -155,7 +158,11 @@ public class InlineLoanCOBExecutorServiceImpl implements
InlineExecutorService<L
}
private List<LoanIdAndLastClosedBusinessDate>
getLoansToBeProcessed(List<Long> loanIds, LocalDate cobBusinessDate) {
- return
loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(cobBusinessDate,
loanIds);
+ List<LoanIdAndLastClosedBusinessDate> loanIdAndLastClosedBusinessDates
= new ArrayList<>();
+ List<List<Long>> partitions = Lists.partition(loanIds,
fineractProperties.getQuery().getInClauseParameterSizeLimit());
+ partitions.forEach(partition -> loanIdAndLastClosedBusinessDates
+
.addAll(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(cobBusinessDate,
partition)));
+ return loanIdAndLastClosedBusinessDates;
}
private List<LoanAccountLock> getLoanAccountLocks(List<Long> loanIds) {
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
index 2e5fc9d2a..a28aa4ee8 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
@@ -55,6 +55,8 @@ public class FineractProperties {
private FineractTemplateProperties template;
private FineractJpaProperties jpa;
+ private FineractQueryProperties query;
+
@Getter
@Setter
public static class FineractTenantProperties {
@@ -255,4 +257,12 @@ public class FineractProperties {
private boolean statementLoggingEnabled;
}
+
+ @Getter
+ @Setter
+ public static class FineractQueryProperties {
+
+ private int inClauseParameterSizeLimit;
+ }
+
}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilter.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilter.java
index d8e9a6773..8583f6dec 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilter.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilter.java
@@ -18,8 +18,10 @@
*/
package org.apache.fineract.infrastructure.jobs.filter;
+import com.google.common.collect.Lists;
import java.io.IOException;
import java.math.BigDecimal;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.function.Predicate;
@@ -35,9 +37,11 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.batch.domain.BatchRequest;
import org.apache.fineract.batch.domain.BatchResponse;
+import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate;
import org.apache.fineract.cob.service.InlineLoanCOBExecutorServiceImpl;
import org.apache.fineract.cob.service.LoanAccountLockService;
import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.data.ApiGlobalErrorResponse;
import org.apache.fineract.infrastructure.core.domain.ExternalId;
import org.apache.fineract.infrastructure.core.filters.BatchFilter;
@@ -64,6 +68,7 @@ public class LoanCOBApiFilter extends OncePerRequestFilter
implements BatchFilte
private final PlatformSecurityContext context;
private final InlineLoanCOBExecutorServiceImpl
inlineLoanCOBExecutorService;
private final LoanRepository loanRepository;
+ private final FineractProperties fineractProperties;
private final LoanRescheduleRequestRepository
loanRescheduleRequestRepository;
@@ -141,8 +146,11 @@ public class LoanCOBApiFilter extends OncePerRequestFilter
implements BatchFilte
}
private boolean isLoanBehind(List<Long> loanIds) {
- return CollectionUtils.isNotEmpty(loanRepository
-
.findAllNonClosedLoansBehindByLoanIds(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE),
loanIds));
+ List<LoanIdAndLastClosedBusinessDate> loanIdAndLastClosedBusinessDates
= new ArrayList<>();
+ List<List<Long>> partitions = Lists.partition(loanIds,
fineractProperties.getQuery().getInClauseParameterSizeLimit());
+ partitions.forEach(partition ->
loanIdAndLastClosedBusinessDates.addAll(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(
+
ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE),
partition)));
+ return CollectionUtils.isNotEmpty(loanIdAndLastClosedBusinessDates);
}
private List<Long> calculateRelevantLoanIds(String pathInfo) {
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/portfolio/account/service/AccountTransfersWritePlatformServiceImpl.java
b/fineract-provider/src/main/java/org/apache/fineract/portfolio/account/service/AccountTransfersWritePlatformServiceImpl.java
index e32452f58..ffd011ff0 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/portfolio/account/service/AccountTransfersWritePlatformServiceImpl.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/portfolio/account/service/AccountTransfersWritePlatformServiceImpl.java
@@ -25,15 +25,18 @@ import static
org.apache.fineract.portfolio.account.AccountDetailConstants.toAcc
import static
org.apache.fineract.portfolio.account.api.AccountTransfersApiConstants.transferAmountParamName;
import static
org.apache.fineract.portfolio.account.api.AccountTransfersApiConstants.transferDateParamName;
+import com.google.common.collect.Lists;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import lombok.RequiredArgsConstructor;
import
org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
import org.apache.fineract.infrastructure.core.api.JsonCommand;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
import
org.apache.fineract.infrastructure.core.data.CommandProcessingResultBuilder;
import org.apache.fineract.infrastructure.core.domain.ExternalId;
@@ -86,6 +89,7 @@ public class AccountTransfersWritePlatformServiceImpl
implements AccountTransfer
private final GSIMRepositoy gsimRepository;
private final ConfigurationDomainService configurationDomainService;
private final ExternalIdFactory externalIdFactory;
+ private final FineractProperties fineractProperties;
@Transactional
@Override
@@ -223,11 +227,13 @@ public class AccountTransfersWritePlatformServiceImpl
implements AccountTransfer
@Transactional
public void reverseTransfersWithFromAccountTransactions(final
Collection<Long> fromTransactionIds,
final PortfolioAccountType accountTypeId) {
- List<AccountTransferTransaction> acccountTransfers = null;
+ List<AccountTransferTransaction> acccountTransfers = new ArrayList<>();
if (accountTypeId.isLoanAccount()) {
- acccountTransfers =
this.accountTransferRepository.findByFromLoanTransactions(fromTransactionIds);
+ List<List<Long>> partitions =
Lists.partition(fromTransactionIds.stream().toList(),
+
fineractProperties.getQuery().getInClauseParameterSizeLimit());
+ partitions.forEach(partition ->
acccountTransfers.addAll(this.accountTransferRepository.findByFromLoanTransactions(partition)));
}
- if (acccountTransfers != null && acccountTransfers.size() > 0) {
+ if (acccountTransfers.size() > 0) {
undoTransactions(acccountTransfers);
}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java
b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java
index 08e3cd0ef..2926fba7c 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java
@@ -192,10 +192,6 @@ public interface LoanRepository extends
JpaRepository<Loan, Long>, JpaSpecificat
@Query(FIND_ALL_NON_CLOSED_LOANS_BY_LAST_CLOSED_BUSINESS_DATE)
List<Long>
findAllNonClosedLoanIdsByLastClosedBusinessDate(@Param("businessDate")
LocalDate businessDate);
- @Query(FIND_ALL_NON_CLOSED_LOANS_BEHIND_BY_LOAN_IDS)
- List<LoanIdAndLastClosedBusinessDate>
findAllNonClosedLoansBehindByLoanIds(@Param("cobBusinessDate") LocalDate
cobBusinessDate,
- @Param("loanIds") List<Long> loanIds);
-
@Query(FIND_ALL_NON_CLOSED_LOANS_BEHIND_OR_NULL_BY_LOAN_IDS)
List<LoanIdAndLastClosedBusinessDate>
findAllNonClosedLoansBehindOrNullByLoanIds(@Param("cobBusinessDate") LocalDate
cobBusinessDate,
@Param("loanIds") List<Long> loanIds);
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepositoryWrapper.java
b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepositoryWrapper.java
index f204786f2..07b479af5 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepositoryWrapper.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepositoryWrapper.java
@@ -18,14 +18,16 @@
*/
package org.apache.fineract.portfolio.loanaccount.domain;
+import com.google.common.collect.Lists;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.ExternalId;
import
org.apache.fineract.portfolio.loanaccount.exception.LoanNotFoundException;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -36,14 +38,11 @@ import
org.springframework.transaction.annotation.Transactional;
* </p>
*/
@Service
+@RequiredArgsConstructor
public class LoanRepositoryWrapper {
private final LoanRepository repository;
-
- @Autowired
- public LoanRepositoryWrapper(final LoanRepository repository) {
- this.repository = repository;
- }
+ private final FineractProperties fineractProperties;
@Transactional(readOnly = true)
public Loan findOneWithNotFoundDetection(final Long id) {
@@ -181,8 +180,11 @@ public class LoanRepositoryWrapper {
// Looks like we need complete Data
public List<Loan> findByIdsAndLoanStatusAndLoanType(@Param("ids")
Collection<Long> ids,
@Param("loanStatuses") Collection<Integer> loanStatuses,
@Param("loanTypes") Collection<Integer> loanTypes) {
- List<Loan> loans =
this.repository.findByIdsAndLoanStatusAndLoanType(ids, loanStatuses, loanTypes);
- if (loans != null && loans.size() > 0) {
+ List<Loan> loans = new ArrayList<>();
+ List<List<Long>> partitions = Lists.partition(ids.stream().toList(),
fineractProperties.getQuery().getInClauseParameterSizeLimit());
+ partitions
+ .forEach(partition ->
loans.addAll(this.repository.findByIdsAndLoanStatusAndLoanType(partition,
loanStatuses, loanTypes)));
+ if (loans.size() > 0) {
for (Loan loan : loans) {
loan.initializeLazyCollections();
}
diff --git a/fineract-provider/src/main/resources/application.properties
b/fineract-provider/src/main/resources/application.properties
index f1b158ea3..ec67e4051 100644
--- a/fineract-provider/src/main/resources/application.properties
+++ b/fineract-provider/src/main/resources/application.properties
@@ -42,6 +42,8 @@
fineract.mode.write-enabled=${FINERACT_MODE_WRITE_ENABLED:true}
fineract.mode.batch-worker-enabled=${FINERACT_MODE_BATCH_WORKER_ENABLED:true}
fineract.mode.batch-manager-enabled=${FINERACT_MODE_BATCH_MANAGER_ENABLED:true}
+fineract.query.in-clause-parameter-size-limit=${FINERACT_QUERY_PARAMETER_SIZE:1000}
+
fineract.correlation.enabled=${FINERACT_LOGGING_HTTP_CORRELATION_ID_ENABLED:false}
fineract.correlation.header-name=${FINERACT_LOGGING_HTTP_CORRELATION_ID_HEADER_NAME:X-Correlation-ID}
diff --git
a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/ApplyLoanLockTaskletStepDefinitions.java
b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/ApplyLoanLockTaskletStepDefinitions.java
index fe8030eb9..78b014cff 100644
---
a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/ApplyLoanLockTaskletStepDefinitions.java
+++
b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/ApplyLoanLockTaskletStepDefinitions.java
@@ -29,6 +29,7 @@ import java.util.List;
import org.apache.fineract.cob.domain.LoanAccountLock;
import org.apache.fineract.cob.domain.LoanAccountLockRepository;
import org.apache.fineract.cob.domain.LockOwner;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import org.mockito.ArgumentCaptor;
@@ -42,7 +43,9 @@ public class ApplyLoanLockTaskletStepDefinitions implements
En {
ArgumentCaptor<LoanAccountLock> valueCaptor =
ArgumentCaptor.forClass(LoanAccountLock.class);
private LoanAccountLockRepository accountLockRepository =
mock(LoanAccountLockRepository.class);
- private ApplyLoanLockTasklet applyLoanLockTasklet = new
ApplyLoanLockTasklet(accountLockRepository);
+ private FineractProperties fineractProperties =
mock(FineractProperties.class);
+ private FineractProperties.FineractQueryProperties fineractQueryProperties
= mock(FineractProperties.FineractQueryProperties.class);
+ private ApplyLoanLockTasklet applyLoanLockTasklet = new
ApplyLoanLockTasklet(accountLockRepository, fineractProperties);
private RepeatStatus resultItem;
private StepContribution stepContribution;
@@ -56,12 +59,16 @@ public class ApplyLoanLockTaskletStepDefinitions implements
En {
this.stepContribution = new StepContribution(stepExecution);
if ("error".equals(action)) {
+
lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
+
lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
lenient().when(this.accountLockRepository.findAllByLoanIdIn(Mockito.anyList())).thenThrow(new
RuntimeException("fail"));
} else {
LoanAccountLock lock1 = new LoanAccountLock(1L,
LockOwner.LOAN_COB_CHUNK_PROCESSING);
LoanAccountLock lock2 = new LoanAccountLock(2L,
LockOwner.LOAN_COB_PARTITIONING);
LoanAccountLock lock3 = new LoanAccountLock(3L,
LockOwner.LOAN_INLINE_COB_PROCESSING);
List<LoanAccountLock> accountLocks = List.of(lock1, lock2,
lock3);
+
lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
+
lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
lenient().when(this.accountLockRepository.findAllByLoanIdIn(Mockito.anyList())).thenReturn(accountLocks);
}
diff --git
a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/FetchAndLockLoanStepDefinitions.java
b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/FetchAndLockLoanStepDefinitions.java
index 2a4a16f7b..9f893edff 100644
---
a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/FetchAndLockLoanStepDefinitions.java
+++
b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/FetchAndLockLoanStepDefinitions.java
@@ -35,6 +35,7 @@ import org.apache.fineract.cob.domain.LoanAccountLock;
import org.apache.fineract.cob.domain.LoanAccountLockRepository;
import org.apache.fineract.cob.domain.LockOwner;
import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import org.mockito.Mockito;
@@ -48,6 +49,9 @@ public class FetchAndLockLoanStepDefinitions implements En {
private final LoanAccountLockRepository loanAccountLockRepository =
mock(LoanAccountLockRepository.class);
private final RetrieveLoanIdService retrieveLoanIdService =
mock(RetrieveLoanIdService.class);
+ private final FineractProperties fineractProperties =
mock(FineractProperties.class);
+ private final FineractProperties.FineractQueryProperties
fineractQueryProperties = mock(
+ FineractProperties.FineractQueryProperties.class);
StepContribution contribution;
private FetchAndLockLoanTasklet fetchAndLockLoanTasklet;
private String action;
@@ -65,17 +69,25 @@ public class FetchAndLockLoanStepDefinitions implements En {
lenient().when(retrieveLoanIdService.retrieveLoanIdsNDaysBehind(anyLong(),
any())).thenReturn(Collections.emptyList());
} else if ("good".equals(action)) {
lenient().when(retrieveLoanIdService.retrieveLoanIdsNDaysBehind(anyLong(),
any())).thenReturn(List.of(1L, 2L, 3L));
+
lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
+
lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
lenient().when(loanAccountLockRepository.findAllByLoanIdIn(Mockito.anyList())).thenReturn(Collections.emptyList());
} else if ("soft lock".equals(action)) {
lenient().when(retrieveLoanIdService.retrieveLoanIdsNDaysBehind(anyLong(),
any())).thenReturn(List.of(1L, 2L, 3L));
+
lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
+
lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
lenient().when(loanAccountLockRepository.findAllByLoanIdIn(Mockito.anyList()))
.thenReturn(List.of(new LoanAccountLock(1L,
LockOwner.LOAN_COB_PARTITIONING)));
} else if ("inline cob".equals(action)) {
lenient().when(retrieveLoanIdService.retrieveLoanIdsNDaysBehind(anyLong(),
any())).thenReturn(List.of(1L, 2L, 3L));
+
lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
+
lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
lenient().when(loanAccountLockRepository.findAllByLoanIdIn(Mockito.anyList()))
.thenReturn(List.of(new LoanAccountLock(2L,
LockOwner.LOAN_INLINE_COB_PROCESSING)));
} else if ("chunk processing".equals(action)) {
lenient().when(retrieveLoanIdService.retrieveLoanIdsNDaysBehind(anyLong(),
any())).thenReturn(List.of(1L, 2L, 3L));
+
lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
+
lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
lenient().when(loanAccountLockRepository.findAllByLoanIdIn(Mockito.anyList()))
.thenReturn(List.of(new LoanAccountLock(3L,
LockOwner.LOAN_COB_CHUNK_PROCESSING)));
}
@@ -85,7 +97,7 @@ public class FetchAndLockLoanStepDefinitions implements En {
contribution = new StepContribution(stepExecution);
contribution.getStepExecution().getJobExecution().getExecutionContext().put(LoanCOBConstant.BUSINESS_DATE_PARAMETER_NAME,
LocalDate.now(ZoneId.systemDefault()).toString());
- fetchAndLockLoanTasklet = new
FetchAndLockLoanTasklet(loanAccountLockRepository, retrieveLoanIdService);
+ fetchAndLockLoanTasklet = new
FetchAndLockLoanTasklet(loanAccountLockRepository, retrieveLoanIdService,
fineractProperties);
});
When("FetchAndLockLoanTasklet.execute method executed", () -> {
diff --git
a/fineract-provider/src/test/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImplTest.java
b/fineract-provider/src/test/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImplTest.java
index 407dcfe3d..791b561c4 100644
---
a/fineract-provider/src/test/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImplTest.java
+++
b/fineract-provider/src/test/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImplTest.java
@@ -23,6 +23,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.lang.reflect.InvocationTargetException;
@@ -35,6 +37,7 @@ import
org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate;
import
org.apache.fineract.cob.exceptions.LoanAccountLockCannotBeOverruledException;
import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
import org.apache.fineract.infrastructure.core.api.JsonCommand;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import org.apache.fineract.portfolio.loanaccount.domain.LoanRepository;
@@ -59,6 +62,10 @@ class InlineLoanCOBExecutorServiceImplTest {
private InlineLoanCOBExecutionDataParser dataParser;
@Mock
private LoanRepository loanRepository;
+ @Mock
+ private FineractProperties fineractProperties;
+ @Mock
+ private FineractProperties.FineractQueryProperties fineractQueryProperties;
@Test
void shouldExceptionThrownIfLoanIsAlreadyLocked() {
@@ -72,10 +79,35 @@ class InlineLoanCOBExecutorServiceImplTest {
ThreadLocalContextUtil.setBusinessDates(businessDates);
when(transactionTemplate.execute(any())).thenThrow(new
LoanAccountLockCannotBeOverruledException(""));
+
when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
+ when(dataParser.parseExecution(any())).thenReturn(List.of(1L));
+
when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
when(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(any(),
anyList())).thenReturn(List.of(loan));
assertThrows(LoanAccountLockCannotBeOverruledException.class, () ->
testObj.executeInlineJob(command, "INLINE_LOAN_COB"));
}
+ @Test
+ void shouldListBePartitioned() {
+ JsonCommand command = mock(JsonCommand.class);
+ LoanIdAndLastClosedBusinessDate loan1 =
mock(LoanIdAndLastClosedBusinessDate.class);
+ LoanIdAndLastClosedBusinessDate loan2 =
mock(LoanIdAndLastClosedBusinessDate.class);
+ LoanIdAndLastClosedBusinessDate loan3 =
mock(LoanIdAndLastClosedBusinessDate.class);
+ ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L,
"default", "Default", "Asia/Kolkata", null));
+ HashMap<BusinessDateType, LocalDate> businessDates = new HashMap<>();
+ LocalDate businessDate = LocalDate.now(ZoneId.systemDefault());
+ businessDates.put(BusinessDateType.BUSINESS_DATE, businessDate);
+ businessDates.put(BusinessDateType.COB_DATE,
businessDate.minusDays(1));
+ ThreadLocalContextUtil.setBusinessDates(businessDates);
+
+ when(transactionTemplate.execute(any())).thenThrow(new
LoanAccountLockCannotBeOverruledException(""));
+
when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
+ when(dataParser.parseExecution(any())).thenReturn(List.of(1L, 2L, 3L));
+
when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(2);
+ when(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(any(),
anyList())).thenReturn(List.of(loan1, loan2, loan3));
+ assertThrows(LoanAccountLockCannotBeOverruledException.class, () ->
testObj.executeInlineJob(command, "INLINE_LOAN_COB"));
+ verify(loanRepository,
times(2)).findAllNonClosedLoansBehindOrNullByLoanIds(any(), anyList());
+ }
+
@Test
void shouldOldestCloseBusinessDateReturnWithCorrectDate()
throws NoSuchMethodException, InvocationTargetException,
IllegalAccessException {
diff --git
a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilterTest.java
b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilterTest.java
index b787abffb..5e67a4c39 100644
---
a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilterTest.java
+++
b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilterTest.java
@@ -42,6 +42,7 @@ import
org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate;
import org.apache.fineract.cob.service.InlineLoanCOBExecutorServiceImpl;
import org.apache.fineract.cob.service.LoanAccountLockService;
import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import
org.apache.fineract.infrastructure.security.service.PlatformSecurityContext;
@@ -80,6 +81,10 @@ class LoanCOBApiFilterTest {
private InlineLoanCOBExecutorServiceImpl inlineLoanCOBExecutorService;
@Mock
private LoanRepository loanRepository;
+ @Mock
+ private FineractProperties fineractProperties;
+ @Mock
+ private FineractProperties.FineractQueryProperties fineractQueryProperties;
@Mock
private LoanRescheduleRequestRepository loanRescheduleRequestRepository;
@@ -145,7 +150,9 @@ class LoanCOBApiFilterTest {
given(request.getPathInfo()).willReturn("/loans/invalid2LoanId/charges");
given(request.getMethod()).willReturn(HTTPMethods.POST.value());
given(context.authenticatedUser()).willReturn(appUser);
- given(loanRepository.findAllNonClosedLoansBehindByLoanIds(
+
given(fineractProperties.getQuery()).willReturn(fineractQueryProperties);
+
given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000);
+ given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(
eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)),
anyList()))
.willReturn(Collections.emptyList());
@@ -187,7 +194,9 @@ class LoanCOBApiFilterTest {
given(loanAccountLockService.isLoanHardLocked(2L)).willReturn(false);
given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(false);
given(context.authenticatedUser()).willReturn(appUser);
- given(loanRepository.findAllNonClosedLoansBehindByLoanIds(
+
given(fineractProperties.getQuery()).willReturn(fineractQueryProperties);
+
given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000);
+ given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(
eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)),
anyList()))
.willReturn(Collections.emptyList());
@@ -214,7 +223,9 @@ class LoanCOBApiFilterTest {
given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(false);
given(context.authenticatedUser()).willReturn(appUser);
given(loanRepository.findIdByExternalId(any())).willReturn(2L);
- given(loanRepository.findAllNonClosedLoansBehindByLoanIds(
+
given(fineractProperties.getQuery()).willReturn(fineractQueryProperties);
+
given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000);
+ given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(
eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)),
anyList()))
.willReturn(Collections.emptyList());
@@ -239,11 +250,13 @@ class LoanCOBApiFilterTest {
given(request.getMethod()).willReturn(HTTPMethods.POST.value());
given(loanAccountLockService.isLoanHardLocked(2L)).willReturn(false);
given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(false);
+
given(fineractProperties.getQuery()).willReturn(fineractQueryProperties);
+
given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000);
LoanRescheduleRequest rescheduleRequest =
mock(LoanRescheduleRequest.class);
given(loanRescheduleRequestRepository.getLoanIdByRescheduleRequestId(resourceId)).willReturn(Optional.of(2L));
given(context.authenticatedUser()).willReturn(appUser);
- given(loanRepository.findAllNonClosedLoansBehindByLoanIds(
+ given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(
eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)),
anyList()))
.willReturn(Collections.emptyList());
@@ -288,8 +301,10 @@ class LoanCOBApiFilterTest {
given(request.getPathInfo()).willReturn("/loans/2?command=approve");
given(request.getMethod()).willReturn(HTTPMethods.POST.value());
given(loanAccountLockService.isLoanHardLocked(2L)).willReturn(false);
- given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(false);
- given(loanRepository.findAllNonClosedLoansBehindByLoanIds(
+ given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(true);
+
given(fineractProperties.getQuery()).willReturn(fineractQueryProperties);
+
given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000);
+ given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(
eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)),
anyList()))
.willReturn(Collections.singletonList(result));
@@ -320,7 +335,9 @@ class LoanCOBApiFilterTest {
given(request.getMethod()).willReturn(HTTPMethods.POST.value());
given(loanAccountLockService.isLoanHardLocked(2L)).willReturn(false);
given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(false);
- given(loanRepository.findAllNonClosedLoansBehindByLoanIds(
+
given(fineractProperties.getQuery()).willReturn(fineractQueryProperties);
+
given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000);
+ given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(
eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)),
anyList()))
.willReturn(Collections.emptyList());
