pvillard31 commented on code in PR #11926: URL: https://github.com/apache/nifi/pull/11926#discussion_r4606492510
########## nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/GitBranchStateCache.java: ########## @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.flow.git; + +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.registry.flow.FlowRegistryException; +import org.apache.nifi.registry.flow.FlowRegistryPermissions; +import org.apache.nifi.registry.flow.RegisteredFlow; +import org.apache.nifi.registry.flow.RegisteredFlowVersionInfo; +import org.apache.nifi.registry.flow.git.client.GitRepositoryClient; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Supplier; + +/** + * Remembers, per branch, what was read for each flow file at the head of the branch. + * <p> + * The framework asks for the flow and its latest version for every versioned Process Group on every + * registry synchronization. Without this cache each of those questions costs requests to the Git provider, + * so the cost of a synchronization grows with the number of versioned Process Groups. With it, each question + * costs a check of the branch head, which the repository client makes cheap while the head is unchanged, plus + * requests for the files touched since the previous head. + * <p> + * Each value is read at the head commit of the state that holds it, and a state is used only while the branch head + * is still that commit, so a value is never answered for another head. A state is also used only with the repository + * client instance it was read with, so values read before a change of configuration are never answered after it. + */ +final class GitBranchStateCache { + + private final Supplier<ComponentLog> loggerSupplier; + private final Map<String, BranchState> branchStates = new ConcurrentHashMap<>(); + private final Map<String, Lock> branchLocks = new ConcurrentHashMap<>(); + + GitBranchStateCache(final Supplier<ComponentLog> loggerSupplier) { + this.loggerSupplier = loggerSupplier; + } + + /** + * Gets the state of the branch at its current head. When the head moved since the state was read, the values for + * files touched between the two heads are dropped; when those files cannot be determined, every value is dropped. + * + * @param repositoryClient the repository client + * @param branch the branch + * @return the state of the branch, or empty when the repository client cannot report the branch head + */ + Optional<BranchState> getState(final GitRepositoryClient repositoryClient, final String branch) throws IOException, FlowRegistryException { + final Optional<String> headSha = repositoryClient.getBranchHeadSha(branch); + if (headSha.isEmpty()) { + return Optional.empty(); + } + + final BranchState state = branchStates.get(branch); + if (state != null && state.isReadWith(repositoryClient) && state.headSha.equals(headSha.get())) { + return Optional.of(state); + } + return Optional.of(moveState(repositoryClient, branch, headSha.get())); + } + + // Each call answers for the head it saw. Concurrent calls that saw different heads may move the state back and forth, + // which costs reads but never answers a value for another head. The lock only spares concurrent calls that saw the + // same new head from comparing the heads more than once. + private BranchState moveState(final GitRepositoryClient repositoryClient, final String branch, final String headSha) { + final Lock branchLock = branchLocks.computeIfAbsent(branch, key -> new ReentrantLock()); + branchLock.lock(); Review Comment: The compare runs while the branch lock is held, so UI or REST calls for the same branch wait behind a slow compare. Could the compare run outside the lock, or should we accept this? ########## nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/test/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClientTest.java: ########## @@ -161,10 +174,85 @@ void createBranchUnsupportedThrowsFlowRegistryException() { assertTrue(repositoryClient.getCreatedBranchCommit().isEmpty()); } + @Test + void repeatedQuestionsForUnchangedBranchAreAnsweredFromCache() throws Exception { + final GitRepositoryClient repositoryClient = mockRepositoryClient(new AtomicReference<>("head-1")); + when(repositoryClient.getLatestCommitShaAtCommit(FLOW_PATH, "head-1")).thenReturn(Optional.of("commit-1")); + when(repositoryClient.getContentFromCommit(FLOW_PATH, "head-1")).thenAnswer(invocation -> getFlowContent("Flow 1")); + + final TestGitFlowRegistryClient flowRegistryClient = createFlowRegistryClient(repositoryClient); + final FlowRegistryClientConfigurationContext context = createContext("main", "[.].*"); + + for (int i = 0; i < 3; i++) { + assertEquals(Optional.of("commit-1"), flowRegistryClient.getLatestVersion(context, FLOW_LOCATION)); + final RegisteredFlow flow = flowRegistryClient.getFlow(context, FLOW_LOCATION); + assertEquals("Flow 1", flow.getName()); + assertEquals("bucket-a", flow.getBucketIdentifier()); + assertEquals("main", flow.getBranch()); + } + + verify(repositoryClient, times(6)).getBranchHeadSha("main"); + verify(repositoryClient, times(1)).getLatestCommitShaAtCommit(FLOW_PATH, "head-1"); + verify(repositoryClient, times(1)).getContentFromCommit(FLOW_PATH, "head-1"); + verify(repositoryClient, never()).getLatestCommitSha(anyString(), anyString()); + verify(repositoryClient, never()).getContentFromBranch(anyString(), anyString()); + } + + @Test + void writeIsVisibleOnNextRead() throws Exception { Review Comment: Could writeIsVisibleOnNextRead also read the flow before the write, so it checks that a cached getFlow is dropped after a commit? ########## nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/GitBranchStateCache.java: ########## @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.flow.git; + +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.registry.flow.FlowRegistryException; +import org.apache.nifi.registry.flow.FlowRegistryPermissions; +import org.apache.nifi.registry.flow.RegisteredFlow; +import org.apache.nifi.registry.flow.RegisteredFlowVersionInfo; +import org.apache.nifi.registry.flow.git.client.GitRepositoryClient; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Supplier; + +/** + * Remembers, per branch, what was read for each flow file at the head of the branch. + * <p> + * The framework asks for the flow and its latest version for every versioned Process Group on every + * registry synchronization. Without this cache each of those questions costs requests to the Git provider, + * so the cost of a synchronization grows with the number of versioned Process Groups. With it, each question + * costs a check of the branch head, which the repository client makes cheap while the head is unchanged, plus + * requests for the files touched since the previous head. + * <p> + * Each value is read at the head commit of the state that holds it, and a state is used only while the branch head + * is still that commit, so a value is never answered for another head. A state is also used only with the repository + * client instance it was read with, so values read before a change of configuration are never answered after it. + */ +final class GitBranchStateCache { + + private final Supplier<ComponentLog> loggerSupplier; + private final Map<String, BranchState> branchStates = new ConcurrentHashMap<>(); + private final Map<String, Lock> branchLocks = new ConcurrentHashMap<>(); + + GitBranchStateCache(final Supplier<ComponentLog> loggerSupplier) { + this.loggerSupplier = loggerSupplier; + } + + /** + * Gets the state of the branch at its current head. When the head moved since the state was read, the values for + * files touched between the two heads are dropped; when those files cannot be determined, every value is dropped. + * + * @param repositoryClient the repository client + * @param branch the branch + * @return the state of the branch, or empty when the repository client cannot report the branch head + */ + Optional<BranchState> getState(final GitRepositoryClient repositoryClient, final String branch) throws IOException, FlowRegistryException { + final Optional<String> headSha = repositoryClient.getBranchHeadSha(branch); + if (headSha.isEmpty()) { + return Optional.empty(); + } + + final BranchState state = branchStates.get(branch); + if (state != null && state.isReadWith(repositoryClient) && state.headSha.equals(headSha.get())) { + return Optional.of(state); + } + return Optional.of(moveState(repositoryClient, branch, headSha.get())); + } + + // Each call answers for the head it saw. Concurrent calls that saw different heads may move the state back and forth, + // which costs reads but never answers a value for another head. The lock only spares concurrent calls that saw the + // same new head from comparing the heads more than once. + private BranchState moveState(final GitRepositoryClient repositoryClient, final String branch, final String headSha) { + final Lock branchLock = branchLocks.computeIfAbsent(branch, key -> new ReentrantLock()); + branchLock.lock(); + try { + final BranchState state = branchStates.get(branch); + final boolean previousStateUsable = state != null && state.isReadWith(repositoryClient); + if (previousStateUsable && state.headSha.equals(headSha)) { + return state; + } + + final BranchState currentState = new BranchState(repositoryClient, headSha); + if (previousStateUsable) { + getChangedPaths(repositoryClient, branch, state.headSha, headSha).ifPresent(paths -> currentState.retainUnchanged(state, paths)); + } + branchStates.put(branch, currentState); + return currentState; + } finally { + branchLock.unlock(); + } + } + + // A failed comparison, such as Not Found when the previous head was removed by a force push, leaves the touched + // files unknown, so the new state starts empty + private Optional<Set<String>> getChangedPaths(final GitRepositoryClient repositoryClient, final String branch, final String fromCommitSha, final String toCommitSha) { + try { + return repositoryClient.getChangedPaths(fromCommitSha, toCommitSha); + } catch (final FileNotFoundException e) { + loggerSupplier.get().debug("Paths changed on branch [{}] from [{}] to [{}] not found, dropping every cached value", branch, fromCommitSha, toCommitSha, e); + return Optional.empty(); + } catch (final IOException | FlowRegistryException e) { Review Comment: A RuntimeException from a client, such as UnsupportedOperationException from getLatestCommitShaAtCommit or an error in getChangedPaths, is not mapped to FlowRegistryException, so the sync shows no failure and a failing compare repeats on every call. Should the cache also treat a RuntimeException from getChangedPaths as unknown changes, and wrap the others into FlowRegistryException? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
