pvillard31 commented on code in PR #11926:
URL: https://github.com/apache/nifi/pull/11926#discussion_r4606492510


##########
nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/GitBranchStateCache.java:
##########
@@ -0,0 +1,225 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.nifi.registry.flow.git;
+
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.registry.flow.FlowRegistryException;
+import org.apache.nifi.registry.flow.FlowRegistryPermissions;
+import org.apache.nifi.registry.flow.RegisteredFlow;
+import org.apache.nifi.registry.flow.RegisteredFlowVersionInfo;
+import org.apache.nifi.registry.flow.git.client.GitRepositoryClient;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Supplier;
+
+/**
+ * Remembers, per branch, what was read for each flow file at the head of the 
branch.
+ * <p>
+ * The framework asks for the flow and its latest version for every versioned 
Process Group on every
+ * registry synchronization. Without this cache each of those questions costs 
requests to the Git provider,
+ * so the cost of a synchronization grows with the number of versioned Process 
Groups. With it, each question
+ * costs a check of the branch head, which the repository client makes cheap 
while the head is unchanged, plus
+ * requests for the files touched since the previous head.
+ * <p>
+ * Each value is read at the head commit of the state that holds it, and a 
state is used only while the branch head
+ * is still that commit, so a value is never answered for another head. A 
state is also used only with the repository
+ * client instance it was read with, so values read before a change of 
configuration are never answered after it.
+ */
+final class GitBranchStateCache {
+
+    private final Supplier<ComponentLog> loggerSupplier;
+    private final Map<String, BranchState> branchStates = new 
ConcurrentHashMap<>();
+    private final Map<String, Lock> branchLocks = new ConcurrentHashMap<>();
+
+    GitBranchStateCache(final Supplier<ComponentLog> loggerSupplier) {
+        this.loggerSupplier = loggerSupplier;
+    }
+
+    /**
+     * Gets the state of the branch at its current head. When the head moved 
since the state was read, the values for
+     * files touched between the two heads are dropped; when those files 
cannot be determined, every value is dropped.
+     *
+     * @param repositoryClient the repository client
+     * @param branch the branch
+     * @return the state of the branch, or empty when the repository client 
cannot report the branch head
+     */
+    Optional<BranchState> getState(final GitRepositoryClient repositoryClient, 
final String branch) throws IOException, FlowRegistryException {
+        final Optional<String> headSha = 
repositoryClient.getBranchHeadSha(branch);
+        if (headSha.isEmpty()) {
+            return Optional.empty();
+        }
+
+        final BranchState state = branchStates.get(branch);
+        if (state != null && state.isReadWith(repositoryClient) && 
state.headSha.equals(headSha.get())) {
+            return Optional.of(state);
+        }
+        return Optional.of(moveState(repositoryClient, branch, headSha.get()));
+    }
+
+    // Each call answers for the head it saw. Concurrent calls that saw 
different heads may move the state back and forth,
+    // which costs reads but never answers a value for another head. The lock 
only spares concurrent calls that saw the
+    // same new head from comparing the heads more than once.
+    private BranchState moveState(final GitRepositoryClient repositoryClient, 
final String branch, final String headSha) {
+        final Lock branchLock = branchLocks.computeIfAbsent(branch, key -> new 
ReentrantLock());
+        branchLock.lock();

Review Comment:
   The compare runs while the branch lock is held, so UI or REST calls for the 
same branch wait behind a slow compare. Could the compare run outside the lock, 
or should we accept this?



##########
nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/test/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClientTest.java:
##########
@@ -161,10 +174,85 @@ void createBranchUnsupportedThrowsFlowRegistryException() 
{
         assertTrue(repositoryClient.getCreatedBranchCommit().isEmpty());
     }
 
+    @Test
+    void repeatedQuestionsForUnchangedBranchAreAnsweredFromCache() throws 
Exception {
+        final GitRepositoryClient repositoryClient = mockRepositoryClient(new 
AtomicReference<>("head-1"));
+        when(repositoryClient.getLatestCommitShaAtCommit(FLOW_PATH, 
"head-1")).thenReturn(Optional.of("commit-1"));
+        when(repositoryClient.getContentFromCommit(FLOW_PATH, 
"head-1")).thenAnswer(invocation -> getFlowContent("Flow 1"));
+
+        final TestGitFlowRegistryClient flowRegistryClient = 
createFlowRegistryClient(repositoryClient);
+        final FlowRegistryClientConfigurationContext context = 
createContext("main", "[.].*");
+
+        for (int i = 0; i < 3; i++) {
+            assertEquals(Optional.of("commit-1"), 
flowRegistryClient.getLatestVersion(context, FLOW_LOCATION));
+            final RegisteredFlow flow = flowRegistryClient.getFlow(context, 
FLOW_LOCATION);
+            assertEquals("Flow 1", flow.getName());
+            assertEquals("bucket-a", flow.getBucketIdentifier());
+            assertEquals("main", flow.getBranch());
+        }
+
+        verify(repositoryClient, times(6)).getBranchHeadSha("main");
+        verify(repositoryClient, 
times(1)).getLatestCommitShaAtCommit(FLOW_PATH, "head-1");
+        verify(repositoryClient, times(1)).getContentFromCommit(FLOW_PATH, 
"head-1");
+        verify(repositoryClient, never()).getLatestCommitSha(anyString(), 
anyString());
+        verify(repositoryClient, never()).getContentFromBranch(anyString(), 
anyString());
+    }
+
+    @Test
+    void writeIsVisibleOnNextRead() throws Exception {

Review Comment:
   Could writeIsVisibleOnNextRead also read the flow before the write, so it 
checks that a cached getFlow is dropped after a commit?



##########
nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/GitBranchStateCache.java:
##########
@@ -0,0 +1,225 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.nifi.registry.flow.git;
+
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.registry.flow.FlowRegistryException;
+import org.apache.nifi.registry.flow.FlowRegistryPermissions;
+import org.apache.nifi.registry.flow.RegisteredFlow;
+import org.apache.nifi.registry.flow.RegisteredFlowVersionInfo;
+import org.apache.nifi.registry.flow.git.client.GitRepositoryClient;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Supplier;
+
+/**
+ * Remembers, per branch, what was read for each flow file at the head of the 
branch.
+ * <p>
+ * The framework asks for the flow and its latest version for every versioned 
Process Group on every
+ * registry synchronization. Without this cache each of those questions costs 
requests to the Git provider,
+ * so the cost of a synchronization grows with the number of versioned Process 
Groups. With it, each question
+ * costs a check of the branch head, which the repository client makes cheap 
while the head is unchanged, plus
+ * requests for the files touched since the previous head.
+ * <p>
+ * Each value is read at the head commit of the state that holds it, and a 
state is used only while the branch head
+ * is still that commit, so a value is never answered for another head. A 
state is also used only with the repository
+ * client instance it was read with, so values read before a change of 
configuration are never answered after it.
+ */
+final class GitBranchStateCache {
+
+    private final Supplier<ComponentLog> loggerSupplier;
+    private final Map<String, BranchState> branchStates = new 
ConcurrentHashMap<>();
+    private final Map<String, Lock> branchLocks = new ConcurrentHashMap<>();
+
+    GitBranchStateCache(final Supplier<ComponentLog> loggerSupplier) {
+        this.loggerSupplier = loggerSupplier;
+    }
+
+    /**
+     * Gets the state of the branch at its current head. When the head moved 
since the state was read, the values for
+     * files touched between the two heads are dropped; when those files 
cannot be determined, every value is dropped.
+     *
+     * @param repositoryClient the repository client
+     * @param branch the branch
+     * @return the state of the branch, or empty when the repository client 
cannot report the branch head
+     */
+    Optional<BranchState> getState(final GitRepositoryClient repositoryClient, 
final String branch) throws IOException, FlowRegistryException {
+        final Optional<String> headSha = 
repositoryClient.getBranchHeadSha(branch);
+        if (headSha.isEmpty()) {
+            return Optional.empty();
+        }
+
+        final BranchState state = branchStates.get(branch);
+        if (state != null && state.isReadWith(repositoryClient) && 
state.headSha.equals(headSha.get())) {
+            return Optional.of(state);
+        }
+        return Optional.of(moveState(repositoryClient, branch, headSha.get()));
+    }
+
+    // Each call answers for the head it saw. Concurrent calls that saw 
different heads may move the state back and forth,
+    // which costs reads but never answers a value for another head. The lock 
only spares concurrent calls that saw the
+    // same new head from comparing the heads more than once.
+    private BranchState moveState(final GitRepositoryClient repositoryClient, 
final String branch, final String headSha) {
+        final Lock branchLock = branchLocks.computeIfAbsent(branch, key -> new 
ReentrantLock());
+        branchLock.lock();
+        try {
+            final BranchState state = branchStates.get(branch);
+            final boolean previousStateUsable = state != null && 
state.isReadWith(repositoryClient);
+            if (previousStateUsable && state.headSha.equals(headSha)) {
+                return state;
+            }
+
+            final BranchState currentState = new BranchState(repositoryClient, 
headSha);
+            if (previousStateUsable) {
+                getChangedPaths(repositoryClient, branch, state.headSha, 
headSha).ifPresent(paths -> currentState.retainUnchanged(state, paths));
+            }
+            branchStates.put(branch, currentState);
+            return currentState;
+        } finally {
+            branchLock.unlock();
+        }
+    }
+
+    // A failed comparison, such as Not Found when the previous head was 
removed by a force push, leaves the touched
+    // files unknown, so the new state starts empty
+    private Optional<Set<String>> getChangedPaths(final GitRepositoryClient 
repositoryClient, final String branch, final String fromCommitSha, final String 
toCommitSha) {
+        try {
+            return repositoryClient.getChangedPaths(fromCommitSha, 
toCommitSha);
+        } catch (final FileNotFoundException e) {
+            loggerSupplier.get().debug("Paths changed on branch [{}] from [{}] 
to [{}] not found, dropping every cached value", branch, fromCommitSha, 
toCommitSha, e);
+            return Optional.empty();
+        } catch (final IOException | FlowRegistryException e) {

Review Comment:
   A RuntimeException from a client, such as UnsupportedOperationException from 
getLatestCommitShaAtCommit or an error in getChangedPaths, is not mapped to 
FlowRegistryException, so the sync shows no failure and a failing compare 
repeats on every call. Should the cache also treat a RuntimeException from 
getChangedPaths as unknown changes, and wrap the others into 
FlowRegistryException?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to