This is an automated email from the ASF dual-hosted git repository.
ijokarumawak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new f1ae059 NIFI-6004 PutFile created directory permissions
f1ae059 is described below
commit f1ae05974e305b8487b26a87338d2faf9c2d851f
Author: adyoun2 <[email protected]>
AuthorDate: Thu Feb 7 16:41:48 2019 +0000
NIFI-6004 PutFile created directory permissions
NIFI-6004 Improve testing of PutFile file and directory permissions
NIFI-6004 Typo in regex
NIFI-6004 Updates based on review
This closes #3294.
Signed-off-by: Koji Kawamura <[email protected]>
---
.../apache/nifi/processors/standard/PutFile.java | 128 ++++++++++++++++++---
.../nifi/processors/standard/TestPutFile.java | 116 +++++++++++++++++--
2 files changed, 220 insertions(+), 24 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFile.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFile.java
index 8784c75..73caa69 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFile.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFile.java
@@ -28,6 +28,9 @@ import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.RequiredPermission;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
@@ -58,6 +61,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
@EventDriven
@@ -83,6 +87,28 @@ public class PutFile extends AbstractProcessor {
public static final String FILE_MODIFY_DATE_ATTRIBUTE =
"file.lastModifiedTime";
public static final String FILE_MODIFY_DATE_ATTR_FORMAT =
"yyyy-MM-dd'T'HH:mm:ssZ";
+ public static final Pattern RWX_PATTERN =
Pattern.compile("^([r-][w-])([x-])([r-][w-])([x-])([r-][w-])([x-])$");
+ public static final Pattern NUM_PATTERN = Pattern.compile("^[0-7]{3}$");
+
+ private static final Validator PERMISSIONS_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(String subject, String input,
ValidationContext context) {
+ ValidationResult.Builder vr = new ValidationResult.Builder();
+ if (context.isExpressionLanguagePresent(input)) {
+ return new
ValidationResult.Builder().subject(subject).input(input).explanation("Expression
Language Present").valid(true).build();
+ }
+
+ if (RWX_PATTERN.matcher(input).matches() ||
NUM_PATTERN.matcher(input).matches()) {
+ return vr.valid(true).build();
+ }
+ return vr.valid(false)
+ .subject(subject)
+ .input(input)
+ .explanation("This must be expressed in rwxr-x--- form or
octal triplet form.")
+ .build();
+ }
+ };
+
public static final PropertyDescriptor DIRECTORY = new
PropertyDescriptor.Builder()
.name("Directory")
.description("The directory to which files should be written. You
may use expression language such as /aa/bb/${path}")
@@ -117,13 +143,13 @@ public class PutFile extends AbstractProcessor {
+ "place of denied permissions (e.g. rw-r--r--) or an
octal number (e.g. 644). You may also use expression language such as "
+ "${file.permissions}.")
.required(false)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .addValidator(PERMISSIONS_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
public static final PropertyDescriptor CHANGE_OWNER = new
PropertyDescriptor.Builder()
.name("Owner")
.description("Sets the owner on the output file to the value of
this attribute. You may also use expression language such as "
- + "${file.owner}.")
+ + "${file.owner}. Note on many operating systems Nifi must
be running as a super-user to have the permissions to set the file owner.")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
@@ -208,9 +234,58 @@ public class PutFile extends AbstractProcessor {
final Path tempCopyFile = rootDirPath.resolve("." + filename);
final Path copyFile = rootDirPath.resolve(filename);
+ final String permissions =
context.getProperty(CHANGE_PERMISSIONS).evaluateAttributeExpressions(flowFile).getValue();
+ final String owner =
context.getProperty(CHANGE_OWNER).evaluateAttributeExpressions(flowFile).getValue();
+ final String group =
context.getProperty(CHANGE_GROUP).evaluateAttributeExpressions(flowFile).getValue();
if (!Files.exists(rootDirPath)) {
if (context.getProperty(CREATE_DIRS).asBoolean()) {
- Files.createDirectories(rootDirPath);
+ Path existing = rootDirPath;
+ while (!Files.exists(existing)) {
+ existing = existing.getParent();
+ }
+ if (permissions != null && !permissions.trim().isEmpty()) {
+ try {
+ String perms = stringPermissions(permissions,
true);
+ if (!perms.isEmpty()) {
+ Files.createDirectories(rootDirPath,
PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(perms)));
+ } else {
+ Files.createDirectories(rootDirPath);
+ }
+ } catch (Exception e) {
+ flowFile = session.penalize(flowFile);
+ session.transfer(flowFile, REL_FAILURE);
+ logger.error("Could not set create directory with
permissions {} because {}", new Object[]{permissions, e});
+ return;
+ }
+ } else {
+ Files.createDirectories(rootDirPath);
+ }
+
+ boolean chOwner = owner != null && !owner.trim().isEmpty();
+ boolean chGroup = group != null && !group.trim().isEmpty();
+ if (chOwner || chGroup) {
+ Path currentPath = rootDirPath;
+ while (!currentPath.equals(existing)) {
+ if (chOwner) {
+ try {
+ UserPrincipalLookupService lookupService =
currentPath.getFileSystem().getUserPrincipalLookupService();
+ Files.setOwner(currentPath,
lookupService.lookupPrincipalByName(owner));
+ } catch (Exception e) {
+ logger.warn("Could not set directory owner
to {} because {}", new Object[]{owner, e});
+ }
+ }
+ if (chGroup) {
+ try {
+ UserPrincipalLookupService lookupService =
currentPath.getFileSystem().getUserPrincipalLookupService();
+ PosixFileAttributeView view =
Files.getFileAttributeView(currentPath, PosixFileAttributeView.class);
+
view.setGroup(lookupService.lookupPrincipalByGroupName(group));
+ } catch (Exception e) {
+ logger.warn("Could not set file group to
{} because {}", new Object[]{group, e});
+ }
+ }
+ currentPath = currentPath.getParent();
+ }
+ }
} else {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
@@ -270,10 +345,9 @@ public class PutFile extends AbstractProcessor {
}
}
- final String permissions =
context.getProperty(CHANGE_PERMISSIONS).evaluateAttributeExpressions(flowFile).getValue();
if (permissions != null && !permissions.trim().isEmpty()) {
try {
- String perms = stringPermissions(permissions);
+ String perms = stringPermissions(permissions, false);
if (!perms.isEmpty()) {
Files.setPosixFilePermissions(dotCopyFile,
PosixFilePermissions.fromString(perms));
}
@@ -282,7 +356,6 @@ public class PutFile extends AbstractProcessor {
}
}
- final String owner =
context.getProperty(CHANGE_OWNER).evaluateAttributeExpressions(flowFile).getValue();
if (owner != null && !owner.trim().isEmpty()) {
try {
UserPrincipalLookupService lookupService =
dotCopyFile.getFileSystem().getUserPrincipalLookupService();
@@ -292,7 +365,6 @@ public class PutFile extends AbstractProcessor {
}
}
- final String group =
context.getProperty(CHANGE_GROUP).evaluateAttributeExpressions(flowFile).getValue();
if (group != null && !group.trim().isEmpty()) {
try {
UserPrincipalLookupService lookupService =
dotCopyFile.getFileSystem().getUserPrincipalLookupService();
@@ -345,13 +417,24 @@ public class PutFile extends AbstractProcessor {
.count();
}
- protected String stringPermissions(String perms) {
+ protected String stringPermissions(String perms, boolean directory) {
String permissions = "";
- final Pattern rwxPattern = Pattern.compile("^[rwx-]{9}$");
- final Pattern numPattern = Pattern.compile("\\d+");
- if (rwxPattern.matcher(perms).matches()) {
- permissions = perms;
- } else if (numPattern.matcher(perms).matches()) {
+ Matcher rwx = RWX_PATTERN.matcher(perms);
+ if (rwx.matches()) {
+ if (directory) {
+ // To read or write, directory access will be required
+ StringBuilder permBuilder = new StringBuilder();
+ permBuilder.append("$1");
+ permBuilder.append(rwx.group(1).equals("--") ? "$2" : "x");
+ permBuilder.append("$3");
+ permBuilder.append(rwx.group(3).equals("--") ? "$4" : "x");
+ permBuilder.append("$5");
+ permBuilder.append(rwx.group(5).equals("--") ? "$6" : "x");
+ permissions = rwx.replaceAll(permBuilder.toString());
+ } else {
+ permissions = perms;
+ }
+ } else if (NUM_PATTERN.matcher(perms).matches()) {
try {
int number = Integer.parseInt(perms, 8);
StringBuilder permBuilder = new StringBuilder();
@@ -365,7 +448,7 @@ public class PutFile extends AbstractProcessor {
} else {
permBuilder.append('-');
}
- if ((number & 0x40) > 0) {
+ if (directory || (number & 0x40) > 0) {
permBuilder.append('x');
} else {
permBuilder.append('-');
@@ -383,7 +466,12 @@ public class PutFile extends AbstractProcessor {
if ((number & 0x8) > 0) {
permBuilder.append('x');
} else {
- permBuilder.append('-');
+ if (directory && (number & 0x30) > 0) {
+ // To read or write, directory access will be required
+ permBuilder.append('x');
+ } else {
+ permBuilder.append('-');
+ }
}
if ((number & 0x4) > 0) {
permBuilder.append('r');
@@ -395,15 +483,21 @@ public class PutFile extends AbstractProcessor {
} else {
permBuilder.append('-');
}
- if ((number & 0x8) > 0) {
+ if ((number & 0x1) > 0) {
permBuilder.append('x');
} else {
- permBuilder.append('-');
+ if (directory && (number & 0x6) > 0) {
+ // To read or write, directory access will be required
+ permBuilder.append('x');
+ } else {
+ permBuilder.append('-');
+ }
}
permissions = permBuilder.toString();
} catch (NumberFormatException ignore) {
}
}
+
return permissions;
}
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutFile.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutFile.java
index 51ad7c0..3440c44 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutFile.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutFile.java
@@ -14,24 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.nifi.processors.standard;
-import static org.junit.Assert.assertEquals;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFilePermission;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
-import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.util.TestRunner;
-import org.apache.nifi.util.TestRunners;
-import org.junit.Before;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
public class TestPutFile {
@@ -207,4 +214,99 @@ public class TestPutFile {
assertEquals("Another file", new String(content));
}
+ private TestRunner putFileRunner;
+
+ private final String testFile = "src" + File.separator + "test" +
File.separator + "resources" + File.separator + "hello.txt";
+
+ @Before
+ public void setup() throws IOException{
+
+ putFileRunner = TestRunners.newTestRunner(PutFile.class);
+ putFileRunner.setProperty(PutFile.CHANGE_OWNER,
System.getProperty("user.name"));
+ putFileRunner.setProperty(PutFile.CHANGE_PERMISSIONS, "rw-r-----");
+ putFileRunner.setProperty(PutFile.CREATE_DIRS, "true");
+ putFileRunner.setProperty(PutFile.DIRECTORY,
"target/test/data/out/PutFile/1/2/3/4/5");
+
+ putFileRunner.setValidateExpressionUsage(false);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ emptyTestDirectory();
+ }
+
+ @Test
+ public void testPutFile() throws IOException {
+ emptyTestDirectory();
+
+ Map<String,String> attributes = new HashMap<>();
+ attributes.put("filename", "testfile.txt");
+
+ putFileRunner.enqueue(Paths.get(testFile), attributes);
+ putFileRunner.run();
+
+ putFileRunner.assertTransferCount(PutSFTP.REL_SUCCESS, 1);
+
+ //verify directory exists
+ Path newDirectory =
Paths.get("target/test/data/out/PutFile/1/2/3/4/5");
+ Path newFile = newDirectory.resolve("testfile.txt");
+ Assert.assertTrue("New directory not created.",
newDirectory.toAbsolutePath().toFile().exists());
+ Assert.assertTrue("New File not created.",
newFile.toAbsolutePath().toFile().exists());
+
+ PosixFileAttributeView filePosixAttributeView =
Files.getFileAttributeView(newFile.toAbsolutePath(),
PosixFileAttributeView.class);
+ Assert.assertEquals(System.getProperty("user.name"),
filePosixAttributeView.getOwner().getName());
+ Set<PosixFilePermission> filePermissions =
filePosixAttributeView.readAttributes().permissions();
+
Assert.assertTrue(filePermissions.contains(PosixFilePermission.OWNER_READ));
+
Assert.assertTrue(filePermissions.contains(PosixFilePermission.OWNER_WRITE));
+
Assert.assertFalse(filePermissions.contains(PosixFilePermission.OWNER_EXECUTE));
+
Assert.assertTrue(filePermissions.contains(PosixFilePermission.GROUP_READ));
+
Assert.assertFalse(filePermissions.contains(PosixFilePermission.GROUP_WRITE));
+
Assert.assertFalse(filePermissions.contains(PosixFilePermission.GROUP_EXECUTE));
+
Assert.assertFalse(filePermissions.contains(PosixFilePermission.OTHERS_READ));
+
Assert.assertFalse(filePermissions.contains(PosixFilePermission.OTHERS_WRITE));
+
Assert.assertFalse(filePermissions.contains(PosixFilePermission.OTHERS_EXECUTE));
+
+ PosixFileAttributeView dirPosixAttributeView =
Files.getFileAttributeView(newDirectory.toAbsolutePath(),
PosixFileAttributeView.class);
+ Assert.assertEquals(System.getProperty("user.name"),
dirPosixAttributeView.getOwner().getName());
+ Set<PosixFilePermission> dirPermissions =
dirPosixAttributeView.readAttributes().permissions();
+
Assert.assertTrue(dirPermissions.contains(PosixFilePermission.OWNER_READ));
+
Assert.assertTrue(dirPermissions.contains(PosixFilePermission.OWNER_WRITE));
+
Assert.assertTrue(dirPermissions.contains(PosixFilePermission.OWNER_EXECUTE));
+
Assert.assertTrue(dirPermissions.contains(PosixFilePermission.GROUP_READ));
+
Assert.assertFalse(dirPermissions.contains(PosixFilePermission.GROUP_WRITE));
+
Assert.assertTrue(dirPermissions.contains(PosixFilePermission.GROUP_EXECUTE));
+
Assert.assertFalse(dirPermissions.contains(PosixFilePermission.OTHERS_READ));
+
Assert.assertFalse(dirPermissions.contains(PosixFilePermission.OTHERS_WRITE));
+
Assert.assertFalse(dirPermissions.contains(PosixFilePermission.OTHERS_EXECUTE));
+
+ putFileRunner.clearTransferState();
+ }
+
+
+ private void emptyTestDirectory() throws IOException {
+ Files.walkFileTree(Paths.get("target/test/data/out/PutFile"), new
FileVisitor<Path>() {
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs) throws IOException {
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs) throws IOException {
+ Files.delete(file);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFileFailed(Path file, IOException exc)
throws IOException {
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException
exc) throws IOException {
+ Files.delete(dir);
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ }
}
