This is an automated email from the ASF dual-hosted git repository.
bbende pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 6ed8a18131 NIFI-13955 - Filter out .directories in Git Registry
clients (#9477)
6ed8a18131 is described below
commit 6ed8a18131e10db90b9decd44efa95fa24bec94e
Author: Pierre Villard <[email protected]>
AuthorDate: Mon Nov 4 18:36:10 2024 +0100
NIFI-13955 - Filter out .directories in Git Registry clients (#9477)
* NIFI-13955 - Filter out .directories in Git Registry clients
* Added a filter property and added a unit test
---
.../registry/flow/git/AbstractGitFlowRegistryClient.java | 14 ++++++++++++++
.../apache/nifi/github/GitHubFlowRegistryClientTest.java | 13 ++++++++++++-
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git
a/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java
b/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java
index ac36714786..ad2c25cc61 100644
---
a/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java
+++
b/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java
@@ -60,6 +60,7 @@ import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
@@ -83,6 +84,15 @@ public abstract class AbstractGitFlowRegistryClient extends
AbstractFlowRegistry
.required(false)
.build();
+ public static final PropertyDescriptor DIRECTORY_FILTER_EXCLUDE = new
PropertyDescriptor.Builder()
+ .name("Directory Filter Exclusion")
+ .description("Directories whose names match the given regular
expression will be ignored "
+ + "when listing buckets.")
+ .defaultValue("[.].*")
+ .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
+ .required(true)
+ .build();
+
static final String DEFAULT_BUCKET_NAME = "default";
static final String DEFAULT_BUCKET_KEEP_FILE_PATH = DEFAULT_BUCKET_NAME +
"/.keep";
static final String DEFAULT_BUCKET_KEEP_FILE_CONTENT = "Do Not Delete";
@@ -98,6 +108,7 @@ public abstract class AbstractGitFlowRegistryClient extends
AbstractFlowRegistry
private volatile FlowSnapshotSerializer flowSnapshotSerializer;
private volatile GitRepositoryClient repositoryClient;
+ private volatile Pattern directoryExclusionPattern;
private final AtomicBoolean clientInitialized = new AtomicBoolean(false);
private volatile List<PropertyDescriptor> propertyDescriptors;
@@ -109,6 +120,7 @@ public abstract class AbstractGitFlowRegistryClient extends
AbstractFlowRegistry
final List<PropertyDescriptor> combinedPropertyDescriptors = new
ArrayList<>(createPropertyDescriptors());
combinedPropertyDescriptors.add(REPOSITORY_BRANCH);
combinedPropertyDescriptors.add(REPOSITORY_PATH);
+ combinedPropertyDescriptors.add(DIRECTORY_FILTER_EXCLUDE);
propertyDescriptors =
Collections.unmodifiableList(combinedPropertyDescriptors);
flowSnapshotSerializer = createFlowSnapshotSerializer();
@@ -174,6 +186,7 @@ public abstract class AbstractGitFlowRegistryClient extends
AbstractFlowRegistry
verifyReadPermissions(repositoryClient);
final Set<FlowRegistryBucket> buckets =
repositoryClient.getTopLevelDirectoryNames(branch).stream()
+ .filter(bucketName ->
!directoryExclusionPattern.matcher(bucketName).matches())
.map(bucketName -> createFlowRegistryBucket(repositoryClient,
bucketName))
.collect(Collectors.toSet());
@@ -568,6 +581,7 @@ public abstract class AbstractGitFlowRegistryClient extends
AbstractFlowRegistry
repositoryClient = createRepositoryClient(context);
clientInitialized.set(true);
initializeDefaultBucket(context);
+ directoryExclusionPattern =
Pattern.compile(context.getProperty(DIRECTORY_FILTER_EXCLUDE).getValue());
}
return repositoryClient;
}
diff --git
a/nifi-extension-bundles/nifi-github-bundle/nifi-github-extensions/src/test/java/org/apache/nifi/github/GitHubFlowRegistryClientTest.java
b/nifi-extension-bundles/nifi-github-bundle/nifi-github-extensions/src/test/java/org/apache/nifi/github/GitHubFlowRegistryClientTest.java
index 32a624ff08..79958c9458 100644
---
a/nifi-extension-bundles/nifi-github-bundle/nifi-github-extensions/src/test/java/org/apache/nifi/github/GitHubFlowRegistryClientTest.java
+++
b/nifi-extension-bundles/nifi-github-bundle/nifi-github-extensions/src/test/java/org/apache/nifi/github/GitHubFlowRegistryClientTest.java
@@ -52,6 +52,7 @@ public class GitHubFlowRegistryClientTest {
static final String DEFAULT_REPO_PATH = "some-path";
static final String DEFAULT_REPO_BRANCH = "some-branch";
+ static final String DEFAULT_FILTER = "[.].*";
private GitHubRepositoryClient repositoryClient;
private FlowSnapshotSerializer flowSnapshotSerializer;
@@ -74,7 +75,14 @@ public class GitHubFlowRegistryClientTest {
when(repositoryClient.hasReadPermission()).thenReturn(true);
when(repositoryClient.hasWritePermission()).thenReturn(true);
-
when(repositoryClient.getTopLevelDirectoryNames(anyString())).thenReturn(Set.of("existing-bucket"));
+
when(repositoryClient.getTopLevelDirectoryNames(anyString())).thenReturn(Set.of("existing-bucket",
".github"));
+ }
+
+ @Test
+ public void testDirExclusion() throws IOException, FlowRegistryException {
+ setupClientConfigurationContextWithDefaults();
+ final Set<FlowRegistryBucket> buckets =
flowRegistryClient.getBuckets(clientConfigurationContext, DEFAULT_REPO_BRANCH);
+ assertEquals(buckets.stream().filter(b ->
b.getName().equals(".github")).count(), 0);
}
@Test
@@ -169,6 +177,9 @@ public class GitHubFlowRegistryClientTest {
final PropertyValue branchPropertyValue =
createMockPropertyValue(branch);
when(clientConfigurationContext.getProperty(GitHubFlowRegistryClient.REPOSITORY_BRANCH)).thenReturn(branchPropertyValue);
+
+ final PropertyValue filterPropertyValue =
createMockPropertyValue(DEFAULT_FILTER);
+
when(clientConfigurationContext.getProperty(GitHubFlowRegistryClient.DIRECTORY_FILTER_EXCLUDE)).thenReturn(filterPropertyValue);
}
private void setupClientConfigurationContextWithDefaults() {
