This is an automated email from the ASF dual-hosted git repository.
turcsanyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new b1e7701 NIFI-6128 UnpackContent: Store unpacked file data
b1e7701 is described below
commit b1e77019accec8cb32c41d34838f2225299f72b4
Author: Tamás Bunth <[email protected]>
AuthorDate: Tue Jun 30 15:47:20 2020 +0200
NIFI-6128 UnpackContent: Store unpacked file data
Tar format allows us to archive files with their original permission,
owner, group name and last modification time.
When unpacking with Tar unpacker, these information are stored in new
attributes with names: "file.inner.*". This way, it preserves backward
compatibility when using parallel with GetFile processor (which stores
information in "file.*").
NIFI-6128 Tar unpackContent: assert date of last modification of content is
a valid date format.
NIFI-6128 UnpackContent: use original attributes
In case of tar format:
- Use "file.*" attributes instead of "file.inner.*" (which eventually
lead to overwrite if tar had been fetched with GetFile)
- Store file permission in "rwx" format, instead of integer
representation.
- Also replace SimpleDateFormat with DateTimeFormatter.
Replace generic error with IllegalArgumentException
Also refactor: move permission string to top of file as static constant.
Update test
Remove 'file systems' text
Unpackcontent: Fill file.creationTime attribute
which always holds the very same value as file.lastModifiedTime.
This closes #4370.
Signed-off-by: Peter Turcsanyi <[email protected]>
---
.../nifi/processors/standard/UnpackContent.java | 28 +++++++++++++-
.../nifi/processors/standard/util/FileInfo.java | 15 ++++++++
.../processors/standard/TestUnpackContent.java | 19 +++++++++
.../processors/standard/util/TestFileInfo.java | 45 ++++++++++++++++++++++
4 files changed, 106 insertions(+), 1 deletion(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
index 32e9ff8..f802a5d 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -64,6 +66,7 @@ import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.standard.util.FileInfo;
import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.util.FlowFileUnpackager;
import org.apache.nifi.util.FlowFileUnpackagerV1;
@@ -92,7 +95,12 @@ import org.apache.nifi.util.FlowFileUnpackagerV3;
+ "parent FlowFile"),
@WritesAttribute(attribute = "fragment.count", description = "The number
of unpacked FlowFiles generated from the parent FlowFile"),
@WritesAttribute(attribute = "segment.original.filename ", description =
"The filename of the parent FlowFile. Extensions of .tar, .zip or .pkg are
removed because "
- + "the MergeContent processor automatically adds those extensions
if it is used to rebuild the original FlowFile")})
+ + "the MergeContent processor automatically adds those extensions
if it is used to rebuild the original FlowFile"),
+ @WritesAttribute(attribute = "file.lastModifiedTime", description = "The
date and time that the unpacked file was last modified (tar only)."),
+ @WritesAttribute(attribute = "file.creationTime", description = "The date
and time that the file was created. This attribute holds always the same value
as file.lastModifiedTime (tar only)."),
+ @WritesAttribute(attribute = "file.owner", description = "The owner of the
unpacked file (tar only)"),
+ @WritesAttribute(attribute = "file.group", description = "The group owner
of the unpacked file (tar only)"),
+ @WritesAttribute(attribute = "file.permissions", description = "The
read/write/execute permissions of the unpacked file (tar only)")})
@SeeAlso(MergeContent.class)
public class UnpackContent extends AbstractProcessor {
// attribute keys
@@ -110,6 +118,15 @@ public class UnpackContent extends AbstractProcessor {
public static final String OCTET_STREAM = "application/octet-stream";
+ public static final String FILE_LAST_MODIFIED_TIME_ATTRIBUTE =
"file.lastModifiedTime";
+ public static final String FILE_CREATION_TIME_ATTRIBUTE =
"file.creationTime";
+ public static final String FILE_OWNER_ATTRIBUTE = "file.owner";
+ public static final String FILE_GROUP_ATTRIBUTE = "file.group";
+ public static final String FILE_PERMISSIONS_ATTRIBUTE = "file.permissions";
+
+ public static final String FILE_MODIFIED_DATE_ATTR_FORMAT =
"yyyy-MM-dd'T'HH:mm:ssZ";
+ public static final DateTimeFormatter DATE_TIME_FORMATTER =
DateTimeFormatter.ofPattern(FILE_MODIFIED_DATE_ATTR_FORMAT).withZone(ZoneId.systemDefault());
+
public static final PropertyDescriptor PACKAGING_FORMAT = new
PropertyDescriptor.Builder()
.name("Packaging Format")
.description("The Packaging Format used to create the file")
@@ -298,6 +315,7 @@ public class UnpackContent extends AbstractProcessor {
public void unpack(final ProcessSession session, final FlowFile
source, final List<FlowFile> unpacked) {
final String fragmentId = UUID.randomUUID().toString();
session.read(source, new InputStreamCallback() {
+
@Override
public void process(final InputStream in) throws IOException {
int fragmentCount = 0;
@@ -321,6 +339,14 @@ public class UnpackContent extends AbstractProcessor {
attributes.put(CoreAttributes.ABSOLUTE_PATH.key(), absPathString);
attributes.put(CoreAttributes.MIME_TYPE.key(),
OCTET_STREAM);
+ attributes.put(FILE_PERMISSIONS_ATTRIBUTE,
FileInfo.permissionToString(tarEntry.getMode()));
+ attributes.put(FILE_OWNER_ATTRIBUTE,
String.valueOf(tarEntry.getUserName()));
+ attributes.put(FILE_GROUP_ATTRIBUTE,
String.valueOf(tarEntry.getGroupName()));
+
+ final String timeAsString =
DATE_TIME_FORMATTER.format(tarEntry.getModTime().toInstant());
+
attributes.put(FILE_LAST_MODIFIED_TIME_ATTRIBUTE, timeAsString);
+ attributes.put(FILE_CREATION_TIME_ATTRIBUTE,
timeAsString);
+
attributes.put(FRAGMENT_ID, fragmentId);
attributes.put(FRAGMENT_INDEX,
String.valueOf(++fragmentCount));
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileInfo.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileInfo.java
index 9b33de1..763ad07 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileInfo.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileInfo.java
@@ -44,6 +44,8 @@ public class FileInfo implements Comparable<FileInfo>,
Serializable, ListableEnt
private static final String OWNER = "owner";
private static final String GROUP = "group";
+ private static final char[] PERMISSION_MODIFIER_CHARS =
"xwrxwrxwr".toCharArray();
+
static {
final List<RecordField> recordFields = new ArrayList<>();
recordFields.add(new RecordField(FILENAME,
RecordFieldType.STRING.getDataType(), false));
@@ -217,6 +219,19 @@ public class FileInfo implements Comparable<FileInfo>,
Serializable, ListableEnt
}
}
+ public static String permissionToString(int fileModeOctal) {
+ if (fileModeOctal > 0777 || fileModeOctal < 00) {
+ throw new IllegalArgumentException("Invalid permission numerals");
+ }
+
+ StringBuilder sb = new StringBuilder();
+ for (char p : PERMISSION_MODIFIER_CHARS) {
+ sb.append((fileModeOctal & 1) == 1 ? p : '-');
+ fileModeOctal >>= 1;
+ }
+ return sb.reverse().toString();
+ }
+
@Override
public String getName() {
return getFileName();
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
index 980d293..42cc244 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
@@ -20,11 +20,14 @@ import static
org.apache.nifi.processors.standard.SplitContent.FRAGMENT_COUNT;
import static org.apache.nifi.processors.standard.SplitContent.FRAGMENT_ID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -70,10 +73,26 @@ public class TestUnpackContent {
autoUnpackRunner.assertTransferCount(UnpackContent.REL_FAILURE, 0);
final List<MockFlowFile> unpacked =
unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_SUCCESS);
+
for (final MockFlowFile flowFile : unpacked) {
final String filename =
flowFile.getAttribute(CoreAttributes.FILENAME.key());
final String folder =
flowFile.getAttribute(CoreAttributes.PATH.key());
final Path path = dataPath.resolve(folder).resolve(filename);
+ assertEquals("rw-r--r--",
flowFile.getAttribute("file.permissions"));
+ assertEquals("jmcarey", flowFile.getAttribute("file.owner"));
+ assertEquals("mkpasswd", flowFile.getAttribute("file.group"));
+ String modifiedTimeAsString =
flowFile.getAttribute("file.lastModifiedTime");
+ try {
+
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ").parse(modifiedTimeAsString);
+ } catch (DateTimeParseException e) {
+ fail();
+ }
+ String creationTimeAsString =
flowFile.getAttribute("file.creationTime");
+ try {
+
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ").parse(creationTimeAsString);
+ } catch (DateTimeParseException e) {
+ fail();
+ }
assertTrue(Files.exists(path));
flowFile.assertContentEquals(path.toFile());
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestFileInfo.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestFileInfo.java
new file mode 100644
index 0000000..1ff9bd3
--- /dev/null
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestFileInfo.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.util;
+
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+public class TestFileInfo {
+
+ @Test
+ public void testPermissionModeToString() {
+ String rwxPerm = FileInfo.permissionToString(0567);
+ assertEquals("r-xrw-rwx", rwxPerm);
+
+ rwxPerm = FileInfo.permissionToString(03);
+ assertEquals("-------wx", rwxPerm);
+
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPermissionModeToStringInvalidFourDigits() {
+ FileInfo.permissionToString(01000);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPermissionModeToStringInvalidNegative() {
+ FileInfo.permissionToString(-1);
+ }
+
+}
