[nifi] branch support/nifi-1.x updated: NIFI-12163 This closes #7835. Improved Syslog 5424 Line Handling
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 9bd7b91514 NIFI-12163 This closes #7835. Improved Syslog 5424 Line
Handling
9bd7b91514 is described below
commit 9bd7b91514d066ee997b79fc70af367719c0a348
Author: exceptionfactory
AuthorDate: Tue Oct 3 15:26:09 2023 -0500
NIFI-12163 This closes #7835. Improved Syslog 5424 Line Handling
- Eliminated unused parseEvent method signatures from
StrictSyslog5424Parser in favor of a single String line method
- Eliminated intermediate conversion from String to byte array and back to
String for Syslog Parser
Signed-off-by: Joseph Witt
---
.../syslog/parsers/StrictSyslog5424Parser.java | 83 +++
.../syslog/BaseStrictSyslog5424ParserTest.java | 93 ++
.../nifi/processors/standard/ParseSyslog5424.java | 26 ++
.../org/apache/nifi/syslog/Syslog5424Reader.java | 16 ++--
.../apache/nifi/syslog/Syslog5424RecordReader.java | 18 ++---
.../nifi/syslog/TestSyslog5424RecordReader.java| 44 +-
6 files changed, 61 insertions(+), 219 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
index ed36c9d34f..d2abe2d972 100644
---
a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
+++
b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
@@ -21,29 +21,18 @@ import com.github.palindromicity.syslog.NilPolicy;
import com.github.palindromicity.syslog.StructuredDataPolicy;
import com.github.palindromicity.syslog.SyslogParserBuilder;
import org.apache.nifi.syslog.events.Syslog5424Event;
-import org.apache.nifi.syslog.keyproviders.SyslogPrefixedKeyProvider;
import org.apache.nifi.syslog.utils.NifiStructuredDataPolicy;
import org.apache.nifi.syslog.utils.NilHandlingPolicy;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-
/**
* Parses a Syslog message from a ByteBuffer into a Syslog5424Event instance.
* For 5424 we use simple-syslog-5424 since it parsers out structured data.
*/
public class StrictSyslog5424Parser {
-private Charset charset;
-private com.github.palindromicity.syslog.SyslogParser parser;
-
-public StrictSyslog5424Parser() {
-this(StandardCharsets.UTF_8, NilHandlingPolicy.NULL,
NifiStructuredDataPolicy.FLATTEN, new SyslogPrefixedKeyProvider());
-}
+private final com.github.palindromicity.syslog.SyslogParser parser;
-public StrictSyslog5424Parser(final Charset charset, final
NilHandlingPolicy nilPolicy,
- NifiStructuredDataPolicy
structuredDataPolicy, KeyProvider keyProvider) {
-this.charset = charset;
+public StrictSyslog5424Parser(final NilHandlingPolicy nilPolicy,
+ final NifiStructuredDataPolicy
structuredDataPolicy, final KeyProvider keyProvider) {
parser = new SyslogParserBuilder()
.withNilPolicy(NilPolicy.valueOf(nilPolicy.name()))
.withStructuredDataPolicy(StructuredDataPolicy.valueOf(structuredDataPolicy.name()))
@@ -52,54 +41,17 @@ public class StrictSyslog5424Parser {
}
/**
- * Parses a Syslog5424Event from a {@code ByteBuffer}.
- *
- * @param buffer a {@code ByteBuffer} containing a syslog message
- * @return a Syslog5424Event parsed from the {@code {@code byte array}}
- */
-public Syslog5424Event parseEvent(final ByteBuffer buffer) {
-return parseEvent(buffer, null);
-}
-
-/**
- * Parses a Syslog5424Event from a {@code ByteBuffer}.
- *
- * @param buffer a {@code ByteBuffer} containing a syslog message
- * @param sender the hostname of the syslog server that sent the message
- * @return a Syslog5424Event parsed from the {@code byte array}
- */
-public Syslog5424Event parseEvent(final ByteBuffer buffer, final String
sender) {
-if (buffer == null) {
-return null;
-}
-return parseEvent(bufferToBytes(buffer), sender);
-}
-
-/**
- * Parses a Syslog5424Event from a {@code byte array}.
+ * Parses a Syslog5424Event from a String
*
- * @param bytes a {@code byte array} containing a syslog message
- * @param sender the hostname of the syslog server that sent the message
- * @return a Syslog5424Event parsed from the {@code byte array}
+ * @param line a {@code String} containing a syslog message
+
[nifi] branch main updated: NIFI-12163 This closes #7835. Improved Syslog 5424 Line Handling
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new da4c6f6e25 NIFI-12163 This closes #7835. Improved Syslog 5424 Line
Handling
da4c6f6e25 is described below
commit da4c6f6e253ad0a490f60554ea57c204927432a2
Author: exceptionfactory
AuthorDate: Tue Oct 3 15:26:09 2023 -0500
NIFI-12163 This closes #7835. Improved Syslog 5424 Line Handling
- Eliminated unused parseEvent method signatures from
StrictSyslog5424Parser in favor of a single String line method
- Eliminated intermediate conversion from String to byte array and back to
String for Syslog Parser
Signed-off-by: Joseph Witt
---
.../syslog/parsers/StrictSyslog5424Parser.java | 83 +++
.../syslog/BaseStrictSyslog5424ParserTest.java | 93 ++
.../nifi/processors/standard/ParseSyslog5424.java | 26 ++
.../org/apache/nifi/syslog/Syslog5424Reader.java | 16 ++--
.../apache/nifi/syslog/Syslog5424RecordReader.java | 18 ++---
.../nifi/syslog/TestSyslog5424RecordReader.java| 44 +-
6 files changed, 61 insertions(+), 219 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
index ed36c9d34f..d2abe2d972 100644
---
a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
+++
b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
@@ -21,29 +21,18 @@ import com.github.palindromicity.syslog.NilPolicy;
import com.github.palindromicity.syslog.StructuredDataPolicy;
import com.github.palindromicity.syslog.SyslogParserBuilder;
import org.apache.nifi.syslog.events.Syslog5424Event;
-import org.apache.nifi.syslog.keyproviders.SyslogPrefixedKeyProvider;
import org.apache.nifi.syslog.utils.NifiStructuredDataPolicy;
import org.apache.nifi.syslog.utils.NilHandlingPolicy;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-
/**
* Parses a Syslog message from a ByteBuffer into a Syslog5424Event instance.
* For 5424 we use simple-syslog-5424 since it parsers out structured data.
*/
public class StrictSyslog5424Parser {
-private Charset charset;
-private com.github.palindromicity.syslog.SyslogParser parser;
-
-public StrictSyslog5424Parser() {
-this(StandardCharsets.UTF_8, NilHandlingPolicy.NULL,
NifiStructuredDataPolicy.FLATTEN, new SyslogPrefixedKeyProvider());
-}
+private final com.github.palindromicity.syslog.SyslogParser parser;
-public StrictSyslog5424Parser(final Charset charset, final
NilHandlingPolicy nilPolicy,
- NifiStructuredDataPolicy
structuredDataPolicy, KeyProvider keyProvider) {
-this.charset = charset;
+public StrictSyslog5424Parser(final NilHandlingPolicy nilPolicy,
+ final NifiStructuredDataPolicy
structuredDataPolicy, final KeyProvider keyProvider) {
parser = new SyslogParserBuilder()
.withNilPolicy(NilPolicy.valueOf(nilPolicy.name()))
.withStructuredDataPolicy(StructuredDataPolicy.valueOf(structuredDataPolicy.name()))
@@ -52,54 +41,17 @@ public class StrictSyslog5424Parser {
}
/**
- * Parses a Syslog5424Event from a {@code ByteBuffer}.
- *
- * @param buffer a {@code ByteBuffer} containing a syslog message
- * @return a Syslog5424Event parsed from the {@code {@code byte array}}
- */
-public Syslog5424Event parseEvent(final ByteBuffer buffer) {
-return parseEvent(buffer, null);
-}
-
-/**
- * Parses a Syslog5424Event from a {@code ByteBuffer}.
- *
- * @param buffer a {@code ByteBuffer} containing a syslog message
- * @param sender the hostname of the syslog server that sent the message
- * @return a Syslog5424Event parsed from the {@code byte array}
- */
-public Syslog5424Event parseEvent(final ByteBuffer buffer, final String
sender) {
-if (buffer == null) {
-return null;
-}
-return parseEvent(bufferToBytes(buffer), sender);
-}
-
-/**
- * Parses a Syslog5424Event from a {@code byte array}.
+ * Parses a Syslog5424Event from a String
*
- * @param bytes a {@code byte array} containing a syslog message
- * @param sender the hostname of the syslog server that sent the message
- * @return a Syslog5424Event parsed from the {@code byte array}
+ * @param line a {@code String} containing a syslog message
+ * @return a
[nifi] branch support/nifi-1.x updated (4354072b4a -> 4cb3f50329)
This is an automated email from the ASF dual-hosted git repository. exceptionfactory pushed a change to branch support/nifi-1.x in repository https://gitbox.apache.org/repos/asf/nifi.git from 4354072b4a NIFI-12158 MockProcessSession write methods preserves attributes (#7828) add 4cb3f50329 NIFI-12164 Fixed compilation with Optional method under JDK 8 No new revisions were added by this update. Summary of changes: .../apache/nifi/processors/iceberg/converter/GenericDataConverters.java | 2 +- .../nifi/processors/iceberg/converter/IcebergRecordConverter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
[nifi] branch support/nifi-1.x updated: NIFI-12158 MockProcessSession write methods preserves attributes (#7828)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 4354072b4a NIFI-12158 MockProcessSession write methods preserves
attributes (#7828)
4354072b4a is described below
commit 4354072b4a623f9e565887bef423f9cd13d62814
Author: Eric Secules
AuthorDate: Tue Oct 3 11:40:31 2023 -0700
NIFI-12158 MockProcessSession write methods preserves attributes (#7828)
Co-authored-by: Eric Secules
---
.../java/org/apache/nifi/util/MockProcessSession.java | 13 ++---
.../java/org/apache/nifi/util/TestMockProcessSession.java | 15 +++
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git
a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
index 2b8326ed50..ce0d93094e 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
@@ -927,16 +927,15 @@ public class MockProcessSession implements ProcessSession
{
if (!(flowFile instanceof MockFlowFile)) {
throw new IllegalArgumentException("Cannot export a flow file that
I did not create");
}
-
final MockFlowFile mockFlowFile = validateState(flowFile);
-writeRecursionSet.add(flowFile);
+writeRecursionSet.add(mockFlowFile);
final ByteArrayOutputStream baos = new ByteArrayOutputStream() {
@Override
public void close() throws IOException {
super.close();
writeRecursionSet.remove(mockFlowFile);
-final MockFlowFile newFlowFile = new
MockFlowFile(mockFlowFile.getId(), flowFile);
+final MockFlowFile newFlowFile = new
MockFlowFile(mockFlowFile.getId(), mockFlowFile);
currentVersions.put(newFlowFile.getId(), newFlowFile);
newFlowFile.setData(toByteArray());
@@ -969,12 +968,12 @@ public class MockProcessSession implements ProcessSession
{
}
@Override
-public MockFlowFile write(final FlowFile flowFile, final StreamCallback
callback) {
+public MockFlowFile write(FlowFile flowFile, final StreamCallback
callback) {
+flowFile = validateState(flowFile);
if (callback == null || flowFile == null) {
throw new IllegalArgumentException("argument cannot be null");
}
-final MockFlowFile mock = validateState(flowFile);
-
+final MockFlowFile mock = (MockFlowFile) flowFile;
final ByteArrayInputStream in = new
ByteArrayInputStream(mock.getData());
final ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -987,7 +986,7 @@ public class MockProcessSession implements ProcessSession {
writeRecursionSet.remove(flowFile);
}
-final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(),
flowFile);
+final MockFlowFile newFlowFile = new MockFlowFile(flowFile.getId(),
flowFile);
currentVersions.put(newFlowFile.getId(), newFlowFile);
newFlowFile.setData(out.toByteArray());
diff --git
a/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
b/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
index eefd4a39dd..775bc2f5ed 100644
--- a/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
+++ b/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
+import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
@@ -133,6 +134,20 @@ public class TestMockProcessSession {
assertFalse(ff1.isPenalized());
}
+@Test
+public void testAttributePreservedAfterWrite() throws IOException {
+final Processor processor = new PoorlyBehavedProcessor();
+final MockProcessSession session = new MockProcessSession(new
SharedSessionState(processor, new AtomicLong(0L)), processor, new
MockStateManager(processor));
+FlowFile ff1 = session.createFlowFile("hello, world".getBytes());
+session.putAttribute(ff1, "key1", "val1");
+session.write(ff1).close();
+session.transfer(ff1, PoorlyBehavedProcessor.REL_FAILURE);
+session.commitAsync();
+List output =
session.getFlowFilesForRelationship(PoorlyBehavedProcessor.REL_FAILURE);
+assertEquals(1, output.size());
+output.get(0).assertAttributeEquals("key1", "val1");
+}
+
protected static class PoorlyBehavedProcessor extends AbstractProcessor {
private static final Relationship REL_FAILURE = new
Relationship.Builder()
[nifi] branch main updated: NIFI-12158 MockProcessSession write methods preserves attributes (#7828)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 721628eb95 NIFI-12158 MockProcessSession write methods preserves
attributes (#7828)
721628eb95 is described below
commit 721628eb955f0f52d8ece0db2afe784c0a144008
Author: Eric Secules
AuthorDate: Tue Oct 3 11:40:31 2023 -0700
NIFI-12158 MockProcessSession write methods preserves attributes (#7828)
Co-authored-by: Eric Secules
---
.../java/org/apache/nifi/util/MockProcessSession.java | 13 ++---
.../java/org/apache/nifi/util/TestMockProcessSession.java | 15 +++
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git
a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
index 4ab99220a2..09cc1d3a5f 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
@@ -919,16 +919,15 @@ public class MockProcessSession implements ProcessSession
{
if (!(flowFile instanceof MockFlowFile)) {
throw new IllegalArgumentException("Cannot export a flow file that
I did not create");
}
-
final MockFlowFile mockFlowFile = validateState(flowFile);
-writeRecursionSet.add(flowFile);
+writeRecursionSet.add(mockFlowFile);
final ByteArrayOutputStream baos = new ByteArrayOutputStream() {
@Override
public void close() throws IOException {
super.close();
writeRecursionSet.remove(mockFlowFile);
-final MockFlowFile newFlowFile = new
MockFlowFile(mockFlowFile.getId(), flowFile);
+final MockFlowFile newFlowFile = new
MockFlowFile(mockFlowFile.getId(), mockFlowFile);
currentVersions.put(newFlowFile.getId(), newFlowFile);
newFlowFile.setData(toByteArray());
@@ -961,12 +960,12 @@ public class MockProcessSession implements ProcessSession
{
}
@Override
-public MockFlowFile write(final FlowFile flowFile, final StreamCallback
callback) {
+public MockFlowFile write(FlowFile flowFile, final StreamCallback
callback) {
+flowFile = validateState(flowFile);
if (callback == null || flowFile == null) {
throw new IllegalArgumentException("argument cannot be null");
}
-final MockFlowFile mock = validateState(flowFile);
-
+final MockFlowFile mock = (MockFlowFile) flowFile;
final ByteArrayInputStream in = new
ByteArrayInputStream(mock.getData());
final ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -979,7 +978,7 @@ public class MockProcessSession implements ProcessSession {
writeRecursionSet.remove(flowFile);
}
-final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(),
flowFile);
+final MockFlowFile newFlowFile = new MockFlowFile(flowFile.getId(),
flowFile);
currentVersions.put(newFlowFile.getId(), newFlowFile);
newFlowFile.setData(out.toByteArray());
diff --git
a/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
b/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
index eefd4a39dd..775bc2f5ed 100644
--- a/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
+++ b/nifi-mock/src/test/java/org/apache/nifi/util/TestMockProcessSession.java
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
+import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
@@ -133,6 +134,20 @@ public class TestMockProcessSession {
assertFalse(ff1.isPenalized());
}
+@Test
+public void testAttributePreservedAfterWrite() throws IOException {
+final Processor processor = new PoorlyBehavedProcessor();
+final MockProcessSession session = new MockProcessSession(new
SharedSessionState(processor, new AtomicLong(0L)), processor, new
MockStateManager(processor));
+FlowFile ff1 = session.createFlowFile("hello, world".getBytes());
+session.putAttribute(ff1, "key1", "val1");
+session.write(ff1).close();
+session.transfer(ff1, PoorlyBehavedProcessor.REL_FAILURE);
+session.commitAsync();
+List output =
session.getFlowFilesForRelationship(PoorlyBehavedProcessor.REL_FAILURE);
+assertEquals(1, output.size());
+output.get(0).assertAttributeEquals("key1", "val1");
+}
+
protected static class PoorlyBehavedProcessor extends AbstractProcessor {
private static final Relationship REL_FAILURE = new
Relationship.Builder()
[nifi] branch support/nifi-1.x updated: NIFI-11739 - Add ability to ignore missing fields in PutIceberg
This is an automated email from the ASF dual-hosted git repository.
pvillard pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 8f4d202271 NIFI-11739 - Add ability to ignore missing fields in
PutIceberg
8f4d202271 is described below
commit 8f4d202271727dfb22be8cb522e4c2e4edf473a1
Author: Matt Burgess
AuthorDate: Wed Jun 21 20:39:43 2023 -0400
NIFI-11739 - Add ability to ignore missing fields in PutIceberg
Signed-off-by: Pierre Villard
This closes #7421.
---
.../iceberg/UnmatchedColumnBehavior.java | 56 +
.../iceberg/converter/GenericDataConverters.java | 65 +-
.../iceberg/converter/IcebergRecordConverter.java | 50 -
.../apache/nifi/processors/iceberg/PutIceberg.java | 14 +-
.../processors/iceberg/TestDataFileActions.java| 6 +-
.../iceberg/TestIcebergRecordConverter.java| 246 ++---
.../iceberg/TestPutIcebergWithHadoopCatalog.java | 29 ++-
.../iceberg/TestPutIcebergWithHiveCatalog.java | 36 ++-
8 files changed, 418 insertions(+), 84 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/UnmatchedColumnBehavior.java
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/UnmatchedColumnBehavior.java
new file mode 100644
index 00..094f0daf60
--- /dev/null
+++
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/UnmatchedColumnBehavior.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.nifi.processors.iceberg;
+
+import org.apache.nifi.components.DescribedValue;
+
+public enum UnmatchedColumnBehavior implements DescribedValue {
+IGNORE_UNMATCHED_COLUMN("Ignore Unmatched Columns",
+"Any column in the database that does not have a field in the
document will be assumed to not be required. No notification will be logged"),
+
+WARNING_UNMATCHED_COLUMN("Warn on Unmatched Columns",
+"Any column in the database that does not have a field in the
document will be assumed to not be required. A warning will be logged"),
+
+FAIL_UNMATCHED_COLUMN("Fail on Unmatched Columns",
+"A flow will fail if any column in the database that does not have
a field in the document. An error will be logged");
+
+
+private final String displayName;
+private final String description;
+
+UnmatchedColumnBehavior(final String displayName, final String
description) {
+this.displayName = displayName;
+this.description = description;
+}
+
+@Override
+public String getValue() {
+return name();
+}
+
+@Override
+public String getDisplayName() {
+return displayName;
+}
+
+@Override
+public String getDescription() {
+return description;
+}
+}
diff --git
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/GenericDataConverters.java
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/GenericDataConverters.java
index c8ee7bd171..794d2c5f37 100644
---
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/GenericDataConverters.java
+++
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/GenericDataConverters.java
@@ -21,9 +21,11 @@ import org.apache.commons.lang3.Validate;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
+import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.record.DataType;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
import
[nifi] branch main updated: NIFI-11739 - Add ability to ignore missing fields in PutIceberg
This is an automated email from the ASF dual-hosted git repository.
pvillard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 3f7b1de6b8 NIFI-11739 - Add ability to ignore missing fields in
PutIceberg
3f7b1de6b8 is described below
commit 3f7b1de6b8102674f86feff55704450671a2c02c
Author: Matt Burgess
AuthorDate: Wed Jun 21 20:39:43 2023 -0400
NIFI-11739 - Add ability to ignore missing fields in PutIceberg
Signed-off-by: Pierre Villard
This closes #7421.
---
.../iceberg/UnmatchedColumnBehavior.java | 56 +
.../iceberg/converter/GenericDataConverters.java | 65 +-
.../iceberg/converter/IcebergRecordConverter.java | 50 -
.../apache/nifi/processors/iceberg/PutIceberg.java | 14 +-
.../processors/iceberg/TestDataFileActions.java| 6 +-
.../iceberg/TestIcebergRecordConverter.java| 246 ++---
.../iceberg/TestPutIcebergWithHadoopCatalog.java | 29 ++-
.../iceberg/TestPutIcebergWithHiveCatalog.java | 36 ++-
8 files changed, 418 insertions(+), 84 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/UnmatchedColumnBehavior.java
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/UnmatchedColumnBehavior.java
new file mode 100644
index 00..094f0daf60
--- /dev/null
+++
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/UnmatchedColumnBehavior.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.nifi.processors.iceberg;
+
+import org.apache.nifi.components.DescribedValue;
+
+public enum UnmatchedColumnBehavior implements DescribedValue {
+IGNORE_UNMATCHED_COLUMN("Ignore Unmatched Columns",
+"Any column in the database that does not have a field in the
document will be assumed to not be required. No notification will be logged"),
+
+WARNING_UNMATCHED_COLUMN("Warn on Unmatched Columns",
+"Any column in the database that does not have a field in the
document will be assumed to not be required. A warning will be logged"),
+
+FAIL_UNMATCHED_COLUMN("Fail on Unmatched Columns",
+"A flow will fail if any column in the database that does not have
a field in the document. An error will be logged");
+
+
+private final String displayName;
+private final String description;
+
+UnmatchedColumnBehavior(final String displayName, final String
description) {
+this.displayName = displayName;
+this.description = description;
+}
+
+@Override
+public String getValue() {
+return name();
+}
+
+@Override
+public String getDisplayName() {
+return displayName;
+}
+
+@Override
+public String getDescription() {
+return description;
+}
+}
diff --git
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/GenericDataConverters.java
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/GenericDataConverters.java
index c8ee7bd171..794d2c5f37 100644
---
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/GenericDataConverters.java
+++
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/GenericDataConverters.java
@@ -21,9 +21,11 @@ import org.apache.commons.lang3.Validate;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
+import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.record.DataType;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
import org.apache.nifi.serialization.record.RecordSchema;
[nifi-minifi-cpp] branch main updated (a38b5a2eb -> 329fe5e17)
This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git from a38b5a2eb MINIFICPP-2233 Update bustache to fix gcc13 compatibility add e4c9d3acd MINIFICPP-2219 Refactor utils::StringUtils -> utils::string add 329fe5e17 MINIFICPP-2238 Disable pull request updates to iss...@nifi.apache.org No new revisions were added by this update. Summary of changes: .asf.yaml | 1 - libminifi/include/utils/StringUtils.h | 738 + libminifi/src/utils/ChecksumCalculator.cpp | 12 +- libminifi/src/utils/StringUtils.cpp| 67 +-- libminifi/test/unit/StringUtilsTests.cpp | 687 +-- 5 files changed, 727 insertions(+), 778 deletions(-)
