galovics commented on code in PR #2576: URL: https://github.com/apache/fineract/pull/2576#discussion_r966397271
########## fineract-provider/src/main/java/org/apache/fineract/cob/common/InitialisationTasklet.java: ########## @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.common; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.useradministration.domain.AppUser; +import org.apache.fineract.useradministration.domain.AppUserRepositoryWrapper; +import org.jetbrains.annotations.NotNull; +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.StepExecutionListener; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.authority.mapping.NullAuthoritiesMapper; +import org.springframework.security.core.context.SecurityContextHolder; + +@Slf4j +@RequiredArgsConstructor +public class InitialisationTasklet implements Tasklet, StepExecutionListener { + + private final AppUserRepositoryWrapper userRepository; + private StepExecution stepExecution; Review Comment: Don't see any usage for this. Was it left here accidentally? ########## fineract-provider/src/main/java/org/apache/fineract/cob/listener/LoanItemListener.java: ########## @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.listener; + +import static org.springframework.transaction.TransactionDefinition.PROPAGATION_REQUIRES_NEW; + +import java.util.List; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.cob.domain.LoanAccountLock; +import org.apache.fineract.cob.domain.LoanAccountLockRepository; +import org.apache.fineract.cob.domain.LockOwner; +import org.apache.fineract.cob.exceptions.LoanReadException; +import org.apache.fineract.infrastructure.core.domain.AbstractPersistableCustom; +import org.apache.fineract.infrastructure.core.serialization.ThrowableSerialization; +import org.apache.fineract.portfolio.loanaccount.domain.Loan; +import org.jetbrains.annotations.NotNull; +import org.springframework.batch.core.annotation.OnProcessError; +import org.springframework.batch.core.annotation.OnReadError; +import org.springframework.batch.core.annotation.OnSkipInProcess; +import org.springframework.batch.core.annotation.OnSkipInRead; +import org.springframework.batch.core.annotation.OnSkipInWrite; +import org.springframework.batch.core.annotation.OnWriteError; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallbackWithoutResult; +import org.springframework.transaction.support.TransactionTemplate; + +@Slf4j +@RequiredArgsConstructor +public class LoanItemListener { + + private final LoanAccountLockRepository accountLockRepository; + + private final TransactionTemplate transactionTemplate; + + @OnReadError + public void onReadError(Exception e) { + LoanReadException ee = (LoanReadException) e; + log.warn("Error was triggered during reading of Loan (id={}) due to: {}", ee.getId(), ThrowableSerialization.serialize(e)); + + transactionTemplate.setPropagationBehavior(PROPAGATION_REQUIRES_NEW); + transactionTemplate.execute(new TransactionCallbackWithoutResult() { + + @Override + protected void doInTransactionWithoutResult(@NotNull TransactionStatus status) { Review Comment: Isn't this a duplicated logic in the below methods? Any chance we could extract this similar piece of code into a method and reuse it? ########## fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanItemReader.java: ########## @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.loan; + +import java.util.ArrayList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.cob.exceptions.LoanAccountWasAlreadyLocked; +import org.apache.fineract.cob.exceptions.LoanReadException; +import org.apache.fineract.portfolio.loanaccount.domain.Loan; +import org.apache.fineract.portfolio.loanaccount.domain.LoanRepository; +import org.jetbrains.annotations.NotNull; +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.annotation.AfterStep; +import org.springframework.batch.core.annotation.BeforeStep; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.item.ItemReader; + +@Slf4j +@RequiredArgsConstructor +public class LoanItemReader implements ItemReader<Loan> { + + private final LoanRepository loanRepository; + private List<Long> alreadyLockedAccounts; + private List<Long> remainingData; + private Long loanId; + + @BeforeStep + public void beforeStep(@NotNull StepExecution stepExecution) { + ExecutionContext executionContext = stepExecution.getExecutionContext(); + ExecutionContext jobExecutionContext = stepExecution.getJobExecution().getExecutionContext(); + List<Long> loanIds = (List<Long>) executionContext.get(LoanCOBConstant.LOAN_IDS); + alreadyLockedAccounts = (List<Long>) jobExecutionContext.get(LoanCOBWorkerConfiguration.ALREADY_LOCKED_LOAN_IDS); + remainingData = new ArrayList<>(loanIds); + } + + @Override + public Loan read() throws Exception { + try { + if (remainingData.size() > 0) { + loanId = remainingData.remove(0); + if (alreadyLockedAccounts != null && alreadyLockedAccounts.remove(loanId)) { + throw new LoanAccountWasAlreadyLocked(loanId); + } + + return loanRepository.findById(loanId).orElse(null); Review Comment: Maybe instead of returning a null in case the loanId doesn't correspond to an actual loan, we could throw an exception cause that indicates an inconsistency in data. ########## fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java: ########## @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.loan; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.cob.domain.LoanAccountLock; +import org.apache.fineract.cob.domain.LoanAccountLockRepository; +import org.apache.fineract.cob.domain.LockOwner; +import org.jetbrains.annotations.NotNull; +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.StepExecutionListener; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.repeat.RepeatStatus; + +@Slf4j +@RequiredArgsConstructor +public class ApplyLoanLockTasklet implements Tasklet, StepExecutionListener { + + private final LoanAccountLockRepository accountLockRepository; + private StepExecution stepExecution; + + @Override + public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull ChunkContext chunkContext) throws Exception { + ExecutionContext executionContext = stepExecution.getExecutionContext(); Review Comment: Question, can't we get the stepExecution from the StepContribution? I recall you might have told me already it wasn't working for some reason, but I'm not sure if I remember correctly. ########## fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java: ########## @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.loan; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.cob.domain.LoanAccountLock; +import org.apache.fineract.cob.domain.LoanAccountLockRepository; +import org.apache.fineract.cob.domain.LockOwner; +import org.jetbrains.annotations.NotNull; +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.StepExecutionListener; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.repeat.RepeatStatus; + +@Slf4j +@RequiredArgsConstructor +public class ApplyLoanLockTasklet implements Tasklet, StepExecutionListener { + + private final LoanAccountLockRepository accountLockRepository; + private StepExecution stepExecution; + + @Override + public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull ChunkContext chunkContext) throws Exception { + ExecutionContext executionContext = stepExecution.getExecutionContext(); + List<Long> loanIds = (List<Long>) executionContext.get(LoanCOBConstant.LOAN_IDS); + List<Long> remainingLoanIds = new ArrayList<>(loanIds); + + List<LoanAccountLock> accountLocks = accountLockRepository.findAllByLoanIdIn(remainingLoanIds); + + List<Long> alreadyHardLockedAccountIds = accountLocks.stream() + .filter(e -> LockOwner.LOAN_COB_CHUNK_PROCESSING.equals(e.getLockOwner())).map(LoanAccountLock::getLoanId).toList(); + + List<Long> alreadyUnderProcessingAccountIds = accountLocks.stream() + .filter(e -> LockOwner.LOAN_INLINE_COB_PROCESSING.equals(e.getLockOwner())).map(LoanAccountLock::getLoanId).toList(); + + Map<Long, LoanAccountLock> alreadySoftLockedAccountsMap = accountLocks.stream() + .filter(e -> LockOwner.LOAN_COB_PARTITIONING.equals(e.getLockOwner())) + .collect(Collectors.toMap(LoanAccountLock::getLoanId, Function.identity())); + + remainingLoanIds.removeAll(alreadyHardLockedAccountIds); + remainingLoanIds.removeAll(alreadyUnderProcessingAccountIds); + + for (Long loanId : remainingLoanIds) { + LoanAccountLock loanAccountLock; + if (alreadySoftLockedAccountsMap.containsKey(loanId)) { + loanAccountLock = alreadySoftLockedAccountsMap.get(loanId); + loanAccountLock.setNewLockOwner(LockOwner.LOAN_COB_CHUNK_PROCESSING); + } else { + loanAccountLock = new LoanAccountLock(loanId, LockOwner.LOAN_COB_CHUNK_PROCESSING); Review Comment: Can this even occur? If something wasn't soft locked before (considering it wasn't inline COB locked either), it means that it was already assigned to a worker in the chunk processing but the worker dropped it, probably due to a crash. In that case, the lock stays on the loanId and we don't need to create a new lock row in the table. What do you think? Am I missing something, maybe this is a case I haven't thought about? ########## fineract-provider/src/main/java/org/apache/fineract/cob/common/InitialisationTasklet.java: ########## @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.common; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.useradministration.domain.AppUser; +import org.apache.fineract.useradministration.domain.AppUserRepositoryWrapper; +import org.jetbrains.annotations.NotNull; +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.StepExecutionListener; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.authority.mapping.NullAuthoritiesMapper; +import org.springframework.security.core.context.SecurityContextHolder; + +@Slf4j +@RequiredArgsConstructor +public class InitialisationTasklet implements Tasklet, StepExecutionListener { + + private final AppUserRepositoryWrapper userRepository; + private StepExecution stepExecution; + + @Override + public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull ChunkContext chunkContext) throws Exception { + AppUser user = userRepository.fetchSystemUser(); + UsernamePasswordAuthenticationToken auth = new UsernamePasswordAuthenticationToken(user, user.getPassword(), + new NullAuthoritiesMapper().mapAuthorities(user.getAuthorities())); + SecurityContextHolder.getContext().setAuthentication(auth); + return RepeatStatus.FINISHED; + } + + @Override + public void beforeStep(@NotNull StepExecution stepExecution) { Review Comment: Any reason for the before and afterStep methods? ########## fineract-provider/src/main/java/org/apache/fineract/cob/domain/LoanAccountLock.java: ########## @@ -18,19 +18,22 @@ */ package org.apache.fineract.cob.domain; -import java.time.LocalDateTime; +import java.time.OffsetDateTime; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Id; import javax.persistence.Table; +import javax.persistence.UniqueConstraint; import javax.persistence.Version; import lombok.Getter; import lombok.NoArgsConstructor; +import org.apache.fineract.infrastructure.core.service.DateUtils; @Entity -@Table(name = "m_loan_account_locks") +@Table(name = "m_loan_account_locks", uniqueConstraints = { + @UniqueConstraint(columnNames = { "loan_id", "lock_owner" }, name = "UNIQUE_ACCOUNT_LOCK") }) Review Comment: As we discussed, the loan_id is already unique in the table, any reason to do a composite unique constraint? ########## fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java: ########## @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.loan; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.cob.domain.LoanAccountLock; +import org.apache.fineract.cob.domain.LoanAccountLockRepository; +import org.apache.fineract.cob.domain.LockOwner; +import org.jetbrains.annotations.NotNull; +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.StepExecutionListener; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.repeat.RepeatStatus; + +@Slf4j +@RequiredArgsConstructor +public class ApplyLoanLockTasklet implements Tasklet, StepExecutionListener { + + private final LoanAccountLockRepository accountLockRepository; + private StepExecution stepExecution; + + @Override + public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull ChunkContext chunkContext) throws Exception { + ExecutionContext executionContext = stepExecution.getExecutionContext(); + List<Long> loanIds = (List<Long>) executionContext.get(LoanCOBConstant.LOAN_IDS); + List<Long> remainingLoanIds = new ArrayList<>(loanIds); + + List<LoanAccountLock> accountLocks = accountLockRepository.findAllByLoanIdIn(remainingLoanIds); + + List<Long> alreadyHardLockedAccountIds = accountLocks.stream() + .filter(e -> LockOwner.LOAN_COB_CHUNK_PROCESSING.equals(e.getLockOwner())).map(LoanAccountLock::getLoanId).toList(); + + List<Long> alreadyUnderProcessingAccountIds = accountLocks.stream() + .filter(e -> LockOwner.LOAN_INLINE_COB_PROCESSING.equals(e.getLockOwner())).map(LoanAccountLock::getLoanId).toList(); + + Map<Long, LoanAccountLock> alreadySoftLockedAccountsMap = accountLocks.stream() + .filter(e -> LockOwner.LOAN_COB_PARTITIONING.equals(e.getLockOwner())) + .collect(Collectors.toMap(LoanAccountLock::getLoanId, Function.identity())); + + remainingLoanIds.removeAll(alreadyHardLockedAccountIds); + remainingLoanIds.removeAll(alreadyUnderProcessingAccountIds); + + for (Long loanId : remainingLoanIds) { + LoanAccountLock loanAccountLock; + if (alreadySoftLockedAccountsMap.containsKey(loanId)) { + loanAccountLock = alreadySoftLockedAccountsMap.get(loanId); Review Comment: I guess we could make this a tiny bit more descriptive for the future. Maybe extract this whole if branch into a method and name it like (random idea) `upgradeLock`. ########## fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java: ########## @@ -63,13 +63,13 @@ public class LoanCOBManagerConfiguration { @Bean public LoanCOBPartitioner partitioner() { - return new LoanCOBPartitioner(cobPropertyService, cobBusinessStepService, jobOperator, jobExplorer, retrieveLoanIdService); + return new LoanCOBPartitioner(propertyService, cobBusinessStepService, jobOperator, jobExplorer, retrieveLoanIdService); } @Bean public Step loanCOBStep() { - return stepBuilderFactory.get(JobName.LOAN_COB.name()).partitioner("Loan COB worker", partitioner()).outputChannel(outboundRequests) - .build(); + return stepBuilderFactory.get(JobName.LOAN_COB.name()).partitioner("loanCOBWorkerStep", partitioner()) Review Comment: Do you think we could extract the string `loanCOBWorkerStep` into a constant so it can be referenced from the manager and worker as well (to avoid future maintenance issues with potential renamings) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
