Repository: nifi Updated Branches: refs/heads/master 286a1c880 -> 71499f774
NIFI-5560 This closes #3000. Added Follow SYMLINK support for listFTP & listSFTP and getFTP & getSFTP processors Signed-off-by: joewitt <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/71499f77 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/71499f77 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/71499f77 Branch: refs/heads/master Commit: 71499f774d5c118c58c38be76283a512005f1e92 Parents: 286a1c8 Author: Kumara M S Hemantha <[email protected]> Authored: Thu Sep 13 02:19:32 2018 +0530 Committer: joewitt <[email protected]> Committed: Wed Nov 21 10:53:38 2018 -0500 ---------------------------------------------------------------------- .../java/org/apache/nifi/processors/standard/GetFTP.java | 1 + .../java/org/apache/nifi/processors/standard/GetSFTP.java | 1 + .../java/org/apache/nifi/processors/standard/ListFTP.java | 1 + .../java/org/apache/nifi/processors/standard/ListSFTP.java | 1 + .../apache/nifi/processors/standard/util/FTPTransfer.java | 7 +++++-- .../apache/nifi/processors/standard/util/FileTransfer.java | 8 ++++++++ .../apache/nifi/processors/standard/util/SFTPTransfer.java | 8 ++++++-- 7 files changed, 23 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java index 79d7914..970eaeb 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java @@ -72,6 +72,7 @@ public class GetFTP extends GetFileTransfer { properties.add(FTPTransfer.PATH_FILTER_REGEX); properties.add(FTPTransfer.POLLING_INTERVAL); properties.add(FTPTransfer.RECURSIVE_SEARCH); + properties.add(FTPTransfer.FOLLOW_SYMLINK); properties.add(FTPTransfer.IGNORE_DOTTED_FILES); properties.add(FTPTransfer.DELETE_ORIGINAL); properties.add(FTPTransfer.CONNECTION_TIMEOUT); http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java index 6b891c7..5a3d667 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java @@ -71,6 +71,7 @@ public class GetSFTP extends GetFileTransfer { properties.add(SFTPTransfer.PATH_FILTER_REGEX); properties.add(SFTPTransfer.POLLING_INTERVAL); properties.add(SFTPTransfer.RECURSIVE_SEARCH); + properties.add(SFTPTransfer.FOLLOW_SYMLINK); properties.add(SFTPTransfer.IGNORE_DOTTED_FILES); properties.add(SFTPTransfer.DELETE_ORIGINAL); properties.add(SFTPTransfer.CONNECTION_TIMEOUT); http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java index 2968c85..41e2e80 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java @@ -79,6 +79,7 @@ public class ListFTP extends ListFileTransfer { properties.add(REMOTE_PATH); properties.add(DISTRIBUTED_CACHE_SERVICE); properties.add(FTPTransfer.RECURSIVE_SEARCH); + properties.add(FTPTransfer.FOLLOW_SYMLINK); properties.add(FTPTransfer.FILE_FILTER_REGEX); properties.add(FTPTransfer.PATH_FILTER_REGEX); properties.add(FTPTransfer.IGNORE_DOTTED_FILES); http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java index 2b5d47a..89012b0 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java @@ -83,6 +83,7 @@ public class ListSFTP extends ListFileTransfer { properties.add(REMOTE_PATH); properties.add(DISTRIBUTED_CACHE_SERVICE); properties.add(SFTPTransfer.RECURSIVE_SEARCH); + properties.add(SFTPTransfer.FOLLOW_SYMLINK); properties.add(SFTPTransfer.FILE_FILTER_REGEX); properties.add(SFTPTransfer.PATH_FILTER_REGEX); properties.add(SFTPTransfer.IGNORE_DOTTED_FILES); http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java index e2ddf6b..85fee40 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java @@ -204,6 +204,7 @@ public class FTPTransfer implements FileTransfer { final boolean ignoreDottedFiles = ctx.getProperty(FileTransfer.IGNORE_DOTTED_FILES).asBoolean(); final boolean recurse = ctx.getProperty(FileTransfer.RECURSIVE_SEARCH).asBoolean(); + final boolean symlink = ctx.getProperty(FileTransfer.FOLLOW_SYMLINK).asBoolean(); final String fileFilterRegex = ctx.getProperty(FileTransfer.FILE_FILTER_REGEX).getValue(); final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex); final String pathFilterRegex = ctx.getProperty(FileTransfer.PATH_FILTER_REGEX).getValue(); @@ -255,11 +256,13 @@ public class FTPTransfer implements FileTransfer { final File newFullPath = new File(path, filename); final String newFullForwardPath = newFullPath.getPath().replace("\\", "/"); - if (recurse && file.isDirectory()) { + // if is a directory and we're supposed to recurse + // OR if is a link and we're supposed to follow symlink + if ((recurse && file.isDirectory()) || (symlink && file.isSymbolicLink())) { try { listing.addAll(getListing(newFullForwardPath, depth + 1, maxResults - count)); } catch (final IOException e) { - logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory", e); + logger.error("Unable to get listing from " + newFullForwardPath + "; skipping", e); } } http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java index d500b9d..56dd22d 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java @@ -141,6 +141,14 @@ public interface FileTransfer extends Closeable { .defaultValue("false") .allowableValues("true", "false") .build(); + public static final PropertyDescriptor FOLLOW_SYMLINK = new PropertyDescriptor.Builder() + .name("follow-symlink") + .displayName("Follow symlink") + .description("If true, will pull even symbolic files and also nested symbolic subdirectories; otherwise, will not read symbolic files and will not traverse symbolic link subdirectories") + .required(true) + .defaultValue("false") + .allowableValues("true", "false") + .build(); public static final PropertyDescriptor FILE_FILTER_REGEX = new PropertyDescriptor.Builder() .name("File Filter Regex") .description("Provides a Java Regular Expression for filtering Filenames; if a filter is supplied, only files whose names match that Regular Expression will be fetched") http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java index b418b4f..34bc0e3 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java @@ -188,6 +188,7 @@ public class SFTPTransfer implements FileTransfer { final boolean ignoreDottedFiles = ctx.getProperty(FileTransfer.IGNORE_DOTTED_FILES).asBoolean(); final boolean recurse = ctx.getProperty(FileTransfer.RECURSIVE_SEARCH).asBoolean(); + final boolean symlink = ctx.getProperty(FileTransfer.FOLLOW_SYMLINK).asBoolean(); final String fileFilterRegex = ctx.getProperty(FileTransfer.FILE_FILTER_REGEX).getValue(); final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex); final String pathFilterRegex = ctx.getProperty(FileTransfer.PATH_FILTER_REGEX).getValue(); @@ -211,6 +212,7 @@ public class SFTPTransfer implements FileTransfer { final ChannelSftp sftp = getChannel(null); final boolean isPathMatch = pathFilterMatches; + //subDirs list is used for both 'sub directories' and 'symlinks' final List<LsEntry> subDirs = new ArrayList<>(); try { final LsEntrySelector filter = new LsEntrySelector() { @@ -231,7 +233,8 @@ public class SFTPTransfer implements FileTransfer { } // if is a directory and we're supposed to recurse - if (recurse && entry.getAttrs().isDir()) { + // OR if is a link and we're supposed to follow symlink + if ((recurse && entry.getAttrs().isDir()) || (symlink && entry.getAttrs().isLink())){ subDirs.add(entry); return LsEntrySelector.CONTINUE; } @@ -278,9 +281,10 @@ public class SFTPTransfer implements FileTransfer { try { getListing(newFullForwardPath, depth + 1, maxResults, listing); } catch (final IOException e) { - logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory", e); + logger.error("Unable to get listing from " + newFullForwardPath + "; skipping", e); } } + } private FileInfo newFileInfo(final LsEntry entry, String path) {
