This is an automated email from the ASF dual-hosted git repository.
jstorck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new 3d21059 NIFI-6405 GetHDFSFileInfo ignores files in the root/start
directory
3d21059 is described below
commit 3d21059e95920ded8034e0bdc7d318571ffef54a
Author: Ferenc Szabó <[email protected]>
AuthorDate: Fri Jun 28 10:28:12 2019 +0200
NIFI-6405 GetHDFSFileInfo ignores files in the root/start directory
modifying test cases, fixing file processing.
Testing done:
- Unit tests
- Manual run against a hadoop instance
This closes #3558
---
.../nifi/processors/hadoop/GetHDFSFileInfo.java | 3 +-
.../processors/hadoop/TestGetHDFSFileInfo.java | 49 ++++++++++++++++++----
.../testRecursiveGroupAllToAttributes.json | 2 +-
.../testRecursiveGroupDirToAttributes-mydir.json | 2 +-
.../testRunWithPermissionsExceptionContent.json | 2 +-
5 files changed, 44 insertions(+), 14 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/GetHDFSFileInfo.java
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/GetHDFSFileInfo.java
index cda53f0..3fe249d 100644
---
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/GetHDFSFileInfo.java
+++
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/GetHDFSFileInfo.java
@@ -369,8 +369,7 @@ public class GetHDFSFileInfo extends
AbstractHadoopProcessor {
// Decide what to do with child: if requested FF per object or
per dir - just emit new FF with info in 'o' object
if (vo != null && !statsOnly) {
parent.addChild(vo);
- if (p != null && req.isRecursive
- && vo.isFile() && !vo.isSymlink()) {
+ if (vo.isFile() && !vo.isSymlink()) {
processHDFSObject(context, session, origFF, req, vo,
false);
}
}
diff --git
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestGetHDFSFileInfo.java
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestGetHDFSFileInfo.java
index 8a50041..ca99a49 100644
---
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestGetHDFSFileInfo.java
+++
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestGetHDFSFileInfo.java
@@ -302,8 +302,8 @@ public class TestGetHDFSFileInfo {
mff.assertAttributeEquals("hdfs.owner", "owner");
mff.assertAttributeEquals("hdfs.group", "group");
mff.assertAttributeEquals("hdfs.lastModified", ""+1523456000000L);
- mff.assertAttributeEquals("hdfs.length", ""+500);
- mff.assertAttributeEquals("hdfs.count.files", ""+5);
+ mff.assertAttributeEquals("hdfs.length", ""+900);
+ mff.assertAttributeEquals("hdfs.count.files", ""+9);
mff.assertAttributeEquals("hdfs.count.dirs", ""+10);
mff.assertAttributeEquals("hdfs.replication", ""+3);
mff.assertAttributeEquals("hdfs.permissions", "rwxr-xr-x");
@@ -329,7 +329,7 @@ public class TestGetHDFSFileInfo {
runner.run();
runner.assertTransferCount(GetHDFSFileInfo.REL_ORIGINAL, 0);
- runner.assertTransferCount(GetHDFSFileInfo.REL_SUCCESS, 7);
+ runner.assertTransferCount(GetHDFSFileInfo.REL_SUCCESS, 9);
runner.assertTransferCount(GetHDFSFileInfo.REL_FAILURE, 0);
runner.assertTransferCount(GetHDFSFileInfo.REL_NOT_FOUND, 0);
@@ -342,8 +342,8 @@ public class TestGetHDFSFileInfo {
mff.assertAttributeEquals("hdfs.owner", "owner");
mff.assertAttributeEquals("hdfs.group", "group");
mff.assertAttributeEquals("hdfs.lastModified",
""+1523456000000L);
- mff.assertAttributeEquals("hdfs.length", ""+500);
- mff.assertAttributeEquals("hdfs.count.files", ""+5);
+ mff.assertAttributeEquals("hdfs.length", ""+900);
+ mff.assertAttributeEquals("hdfs.count.files", ""+9);
mff.assertAttributeEquals("hdfs.count.dirs", ""+10);
mff.assertAttributeEquals("hdfs.replication", ""+3);
mff.assertAttributeEquals("hdfs.permissions", "rwxr-xr-x");
@@ -426,11 +426,37 @@ public class TestGetHDFSFileInfo {
mff.assertAttributeEquals("hdfs.replication", ""+3);
mff.assertAttributeEquals("hdfs.permissions", "rw-r--r--");
mff.assertAttributeNotExists("hdfs.status");
+ }else if (mff.getAttribute("hdfs.objectName").equals("regFile4")) {
+ matchCount++;
+ mff.assertAttributeEquals("hdfs.path", "/some/home/mydir");
+ mff.assertAttributeEquals("hdfs.type", "file");
+ mff.assertAttributeEquals("hdfs.owner", "owner");
+ mff.assertAttributeEquals("hdfs.group", "group");
+ mff.assertAttributeEquals("hdfs.lastModified",
""+1523456000000L);
+ mff.assertAttributeEquals("hdfs.length", ""+100);
+ mff.assertAttributeNotExists("hdfs.count.files");
+ mff.assertAttributeNotExists("hdfs.count.dirs");
+ mff.assertAttributeEquals("hdfs.replication", ""+3);
+ mff.assertAttributeEquals("hdfs.permissions", "rw-r--r--");
+ mff.assertAttributeNotExists("hdfs.status");
+ }else if
(mff.getAttribute("hdfs.objectName").equals("regFile5")) {
+ matchCount++;
+ mff.assertAttributeEquals("hdfs.path", "/some/home/mydir");
+ mff.assertAttributeEquals("hdfs.type", "file");
+ mff.assertAttributeEquals("hdfs.owner", "owner");
+ mff.assertAttributeEquals("hdfs.group", "group");
+ mff.assertAttributeEquals("hdfs.lastModified",
""+1523456000000L);
+ mff.assertAttributeEquals("hdfs.length", ""+100);
+ mff.assertAttributeNotExists("hdfs.count.files");
+ mff.assertAttributeNotExists("hdfs.count.dirs");
+ mff.assertAttributeEquals("hdfs.replication", ""+3);
+ mff.assertAttributeEquals("hdfs.permissions", "rw-r--r--");
+ mff.assertAttributeNotExists("hdfs.status");
}else {
runner.assertNotValid();
}
}
- Assert.assertEquals(matchCount, 7);
+ Assert.assertEquals(matchCount, 9);
}
@Test
@@ -462,8 +488,8 @@ public class TestGetHDFSFileInfo {
mff.assertAttributeEquals("hdfs.owner", "owner");
mff.assertAttributeEquals("hdfs.group", "group");
mff.assertAttributeEquals("hdfs.lastModified",
""+1523456000000L);
- mff.assertAttributeEquals("hdfs.length", ""+500);
- mff.assertAttributeEquals("hdfs.count.files", ""+5);
+ mff.assertAttributeEquals("hdfs.length", ""+900);
+ mff.assertAttributeEquals("hdfs.count.files", ""+9);
mff.assertAttributeEquals("hdfs.count.dirs", ""+10);
mff.assertAttributeEquals("hdfs.replication", ""+3);
mff.assertAttributeEquals("hdfs.permissions", "rwxr-xr-x");
@@ -540,9 +566,14 @@ public class TestGetHDFSFileInfo {
/*
* For all basic tests, this provides a structure of files in dirs:
* Total number of dirs: 9 (1 root, 4 dotted)
- * Total number of files: 4 (2 dotted)
+ * Total number of files: 8 (4 dotted)
*/
protected void setFileSystemBasicTree(final MockFileSystem fs) {
+ fs.addFileStatus(fs.newDir("/some/home/mydir"),
fs.newFile("/some/home/mydir/regFile4"));
+ fs.addFileStatus(fs.newDir("/some/home/mydir"),
fs.newFile("/some/home/mydir/regFile5"));
+ fs.addFileStatus(fs.newDir("/some/home/mydir"),
fs.newFile("/some/home/mydir/.dotFile4"));
+ fs.addFileStatus(fs.newDir("/some/home/mydir"),
fs.newFile("/some/home/mydir/.dotFile5"));
+
fs.addFileStatus(fs.newDir("/some/home/mydir"),
fs.newDir("/some/home/mydir/dir1"));
fs.addFileStatus(fs.newDir("/some/home/mydir"),
fs.newDir("/some/home/mydir/dir2"));
fs.addFileStatus(fs.newDir("/some/home/mydir"),
fs.newDir("/some/home/mydir/.dir3"));
diff --git
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRecursiveGroupAllToAttributes.json
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRecursiveGroupAllToAttributes.json
index 096f244..df3d60a 100644
---
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRecursiveGroupAllToAttributes.json
+++
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRecursiveGroupAllToAttributes.json
@@ -1 +1 @@
-{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"500","replication":"3","countFiles":"5","countDirs":"10","permissions":"rwxr-xr-x","content":[{"objectName":"dir1","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x","content":[{"objectName":"regFile","path":"
[...]
\ No newline at end of file
+{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"900","replication":"3","countFiles":"9","countDirs":"10","permissions":"rwxr-xr-x","content":[{"objectName":"regFile5","path":"/some/home/mydir","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"regFile4","path":"/some/home/mydir","type":"file","owner":"ow
[...]
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRecursiveGroupDirToAttributes-mydir.json
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRecursiveGroupDirToAttributes-mydir.json
index e1e7d63..0267caf 100644
---
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRecursiveGroupDirToAttributes-mydir.json
+++
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRecursiveGroupDirToAttributes-mydir.json
@@ -1 +1 @@
-{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"500","replication":"3","countFiles":"5","countDirs":"10","permissions":"rwxr-xr-x","content":[{"objectName":"dir1","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x"},{"objectName":"dir2","path":"/some/home/my
[...]
\ No newline at end of file
+{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"900","replication":"3","countFiles":"9","countDirs":"10","permissions":"rwxr-xr-x","content":[{"objectName":"regFile5","path":"/some/home/mydir","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"regFile4","path":"/some/home/mydir","type":"file","owner":"ow
[...]
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRunWithPermissionsExceptionContent.json
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRunWithPermissionsExceptionContent.json
index b21cf04..dfb53d2 100644
---
a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRunWithPermissionsExceptionContent.json
+++
b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRunWithPermissionsExceptionContent.json
@@ -1 +1 @@
-{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"500","replication":"3","countFiles":"5","countDirs":"11","permissions":"rwxr-xr-x","content":[{"objectName":"dir1","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"4","permissions":"rwxr-xr-x","content":[{"objectName":"list_exception_ja
[...]
\ No newline at end of file
+{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"900","replication":"3","countFiles":"9","countDirs":"11","permissions":"rwxr-xr-x","content":[{"objectName":"regFile5","path":"/some/home/mydir","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"regFile4","path":"/some/home/mydir","type":"file","owner":"ow
[...]
\ No newline at end of file
