[nifi] branch main updated: NIFI-11540 Removed legacy segment attributes from MergeContent
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 8382b0a7a3 NIFI-11540 Removed legacy segment attributes from
MergeContent
8382b0a7a3 is described below
commit 8382b0a7a3a2735e9047db660df4033c9976c911
Author: Jeyassri Balchandran
AuthorDate: Fri May 26 19:13:46 2023 +0530
NIFI-11540 Removed legacy segment attributes from MergeContent
This closes #7305
Co-authored-by: David Handermann
Signed-off-by: David Handermann
---
.../nifi/processors/standard/MergeContent.java | 23 ++
.../nifi/processors/standard/SegmentContent.java | 20 ---
.../nifi/processors/standard/TestMergeContent.java | 12 +--
3 files changed, 8 insertions(+), 47 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
index c0ce6ade8a..5b4afdca81 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
@@ -154,14 +154,8 @@ public class MergeContent extends BinFiles {
public static final String FRAGMENT_ID_ATTRIBUTE =
FragmentAttributes.FRAGMENT_ID.key();
public static final String FRAGMENT_INDEX_ATTRIBUTE =
FragmentAttributes.FRAGMENT_INDEX.key();
public static final String FRAGMENT_COUNT_ATTRIBUTE =
FragmentAttributes.FRAGMENT_COUNT.key();
-
-// old style attributes
-public static final String SEGMENT_ID_ATTRIBUTE = "segment.identifier";
-public static final String SEGMENT_INDEX_ATTRIBUTE = "segment.index";
-public static final String SEGMENT_COUNT_ATTRIBUTE = "segment.count";
public static final String SEGMENT_ORIGINAL_FILENAME =
FragmentAttributes.SEGMENT_ORIGINAL_FILENAME.key();
-
public static final AllowableValue METADATA_STRATEGY_USE_FIRST = new
AllowableValue("Use First Metadata", "Use First Metadata",
"For any input format that supports metadata (Avro, e.g.), the
metadata for the first FlowFile in the bin will be set on the output
FlowFile.");
@@ -184,8 +178,7 @@ public class MergeContent extends BinFiles {
"Defragment",
"Defragment",
"Combines fragments that are associated by attributes back into a
single cohesive FlowFile. If using this strategy, all FlowFiles must "
-+ "have the attributes , ,
and or alternatively (for backward compatibility "
-+ "purposes) , , and
. All FlowFiles with the same value for \"fragment.identifier\" "
++ "have the attributes , ,
and . All FlowFiles with the same value for
\"fragment.identifier\" "
+ "will be grouped together. All FlowFiles in this group must have
the same value for the \"fragment.count\" attribute. All FlowFiles "
+ "in this group must have a unique value for the
\"fragment.index\" attribute between 0 and the value of the \"fragment.count\"
attribute.");
@@ -430,19 +423,7 @@ public class MergeContent extends BinFiles {
@Override
protected FlowFile preprocessFlowFile(final ProcessContext context, final
ProcessSession session, final FlowFile flowFile) {
-FlowFile processed = flowFile;
-// handle backward compatibility with old segment attributes
-if (processed.getAttribute(FRAGMENT_COUNT_ATTRIBUTE) == null &&
processed.getAttribute(SEGMENT_COUNT_ATTRIBUTE) != null) {
-processed = session.putAttribute(processed,
FRAGMENT_COUNT_ATTRIBUTE, processed.getAttribute(SEGMENT_COUNT_ATTRIBUTE));
-}
-if (processed.getAttribute(FRAGMENT_INDEX_ATTRIBUTE) == null &&
processed.getAttribute(SEGMENT_INDEX_ATTRIBUTE) != null) {
-processed = session.putAttribute(processed,
FRAGMENT_INDEX_ATTRIBUTE, processed.getAttribute(SEGMENT_INDEX_ATTRIBUTE));
-}
-if (processed.getAttribute(FRAGMENT_ID_ATTRIBUTE) == null &&
processed.getAttribute(SEGMENT_ID_ATTRIBUTE) != null) {
-processed = session.putAttribute(processed, FRAGMENT_ID_ATTRIBUTE,
processed.getAttribute(SEGMENT_ID_ATTRIBUTE));
-}
-
-return processed;
+return flowFile;
}
@Override
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SegmentContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SegmentContent.java
index d29bff812d..8dba735ce1 100644
[nifi] branch support/nifi-1.x updated: NIFI-11553 Add Record handling and more Properties for GCP PubSub
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 99610f09a4 NIFI-11553 Add Record handling and more Properties for GCP
PubSub
99610f09a4 is described below
commit 99610f09a40513b29574c175b4fbcf6bba4d7294
Author: Paul Grey
AuthorDate: Tue May 16 09:38:39 2023 -0400
NIFI-11553 Add Record handling and more Properties for GCP PubSub
- Added Message Derivation Strategy to PublishGCPubSub with FlowFile and
Record options
- Added API Endpoint property to PublishGCPPubSub and ConsumeGCPubSub
- Added Batch configuration properties
This closes #7274
Signed-off-by: David Handermann
(cherry picked from commit 336b857442109e1f2c8b3b6108106514609159f5)
---
.../gcp/pubsub/AbstractGCPubSubProcessor.java | 39 ++-
.../processors/gcp/pubsub/ConsumeGCPubSub.java | 13 +-
.../processors/gcp/pubsub/PubSubAttributes.java| 3 +
.../processors/gcp/pubsub/PublishGCPubSub.java | 320 -
.../gcp/pubsub/lite/PublishGCPubSubLite.java | 24 +-
.../gcp/pubsub/publish/FlowFileResult.java | 143 +
.../pubsub/publish/MessageDerivationStrategy.java | 53
.../pubsub/publish/TrackedApiFutureCallback.java | 45 +++
.../processors/gcp/pubsub/ConsumeGCPubSubIT.java | 6 +-
.../processors/gcp/pubsub/PublishGCPubSubIT.java | 4 +-
.../processors/gcp/pubsub/PublishGCPubSubTest.java | 233 +++
.../gcp/pubsub/lite/PublishGCPubSubLiteTest.java | 65 +
.../src/test/resources/pubsub/records.json | 17 ++
13 files changed, 859 insertions(+), 106 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubProcessor.java
b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubProcessor.java
index fc135dba81..6602a0e0c8 100644
---
a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubProcessor.java
+++
b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubProcessor.java
@@ -19,9 +19,11 @@ package org.apache.nifi.processors.gcp.pubsub;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.ServiceOptions;
+import com.google.cloud.pubsub.v1.stub.PublisherStubSettings;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.VerifiableProcessor;
@@ -36,9 +38,9 @@ import java.util.Set;
public abstract class AbstractGCPubSubProcessor extends AbstractGCPProcessor
implements VerifiableProcessor {
-public static final PropertyDescriptor BATCH_SIZE = new
PropertyDescriptor.Builder()
+public static final PropertyDescriptor BATCH_SIZE_THRESHOLD = new
PropertyDescriptor.Builder()
.name("gcp-pubsub-publish-batch-size")
-.displayName("Batch Size")
+.displayName("Batch Size Threshold")
.description("Indicates the number of messages the cloud service
should bundle together in a batch. If not set and left empty, only one message
" +
"will be used in a batch")
.required(true)
@@ -46,6 +48,39 @@ public abstract class AbstractGCPubSubProcessor extends
AbstractGCPProcessor imp
.addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
.build();
+public static final PropertyDescriptor BATCH_BYTES_THRESHOLD = new
PropertyDescriptor.Builder()
+.name("gcp-batch-bytes")
+.displayName("Batch Bytes Threshold")
+.description("Publish request gets triggered based on this Batch
Bytes Threshold property and"
++ " the " + BATCH_SIZE_THRESHOLD.getDisplayName() + "
property, whichever condition is met first.")
+.required(true)
+.defaultValue("3 MB")
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+.addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor BATCH_DELAY_THRESHOLD = new
PropertyDescriptor.Builder()
+.name("gcp-pubsub-publish-batch-delay")
+.displayName("Batch Delay Threshold")
+.description("Indicates the delay threshold to use for batching.
After this amount of time has elapsed " +
+"(counting from the
[nifi] branch main updated: NIFI-11553 Add Record handling and more Properties for GCP PubSub
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 336b857442 NIFI-11553 Add Record handling and more Properties for GCP
PubSub
336b857442 is described below
commit 336b857442109e1f2c8b3b6108106514609159f5
Author: Paul Grey
AuthorDate: Tue May 16 09:38:39 2023 -0400
NIFI-11553 Add Record handling and more Properties for GCP PubSub
- Added Message Derivation Strategy to PublishGCPubSub with FlowFile and
Record options
- Added API Endpoint property to PublishGCPPubSub and ConsumeGCPubSub
- Added Batch configuration properties
This closes #7274
Signed-off-by: David Handermann
---
.../gcp/pubsub/AbstractGCPubSubProcessor.java | 39 ++-
.../processors/gcp/pubsub/ConsumeGCPubSub.java | 13 +-
.../processors/gcp/pubsub/PubSubAttributes.java| 3 +
.../processors/gcp/pubsub/PublishGCPubSub.java | 320 -
.../gcp/pubsub/lite/PublishGCPubSubLite.java | 24 +-
.../gcp/pubsub/publish/FlowFileResult.java | 143 +
.../pubsub/publish/MessageDerivationStrategy.java | 53
.../pubsub/publish/TrackedApiFutureCallback.java | 45 +++
.../processors/gcp/pubsub/ConsumeGCPubSubIT.java | 6 +-
.../processors/gcp/pubsub/PublishGCPubSubIT.java | 4 +-
.../processors/gcp/pubsub/PublishGCPubSubTest.java | 233 +++
.../gcp/pubsub/lite/PublishGCPubSubLiteTest.java | 65 +
.../src/test/resources/pubsub/records.json | 17 ++
13 files changed, 859 insertions(+), 106 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubProcessor.java
b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubProcessor.java
index fc135dba81..6602a0e0c8 100644
---
a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubProcessor.java
+++
b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubProcessor.java
@@ -19,9 +19,11 @@ package org.apache.nifi.processors.gcp.pubsub;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.ServiceOptions;
+import com.google.cloud.pubsub.v1.stub.PublisherStubSettings;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.VerifiableProcessor;
@@ -36,9 +38,9 @@ import java.util.Set;
public abstract class AbstractGCPubSubProcessor extends AbstractGCPProcessor
implements VerifiableProcessor {
-public static final PropertyDescriptor BATCH_SIZE = new
PropertyDescriptor.Builder()
+public static final PropertyDescriptor BATCH_SIZE_THRESHOLD = new
PropertyDescriptor.Builder()
.name("gcp-pubsub-publish-batch-size")
-.displayName("Batch Size")
+.displayName("Batch Size Threshold")
.description("Indicates the number of messages the cloud service
should bundle together in a batch. If not set and left empty, only one message
" +
"will be used in a batch")
.required(true)
@@ -46,6 +48,39 @@ public abstract class AbstractGCPubSubProcessor extends
AbstractGCPProcessor imp
.addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
.build();
+public static final PropertyDescriptor BATCH_BYTES_THRESHOLD = new
PropertyDescriptor.Builder()
+.name("gcp-batch-bytes")
+.displayName("Batch Bytes Threshold")
+.description("Publish request gets triggered based on this Batch
Bytes Threshold property and"
++ " the " + BATCH_SIZE_THRESHOLD.getDisplayName() + "
property, whichever condition is met first.")
+.required(true)
+.defaultValue("3 MB")
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+.addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor BATCH_DELAY_THRESHOLD = new
PropertyDescriptor.Builder()
+.name("gcp-pubsub-publish-batch-delay")
+.displayName("Batch Delay Threshold")
+.description("Indicates the delay threshold to use for batching.
After this amount of time has elapsed " +
+"(counting from the first element added), the elements
will be wrapped up in a batch and sent. " +
+
[nifi] branch support/nifi-1.x updated: NIFI-11679 Refactored EncryptedRepoContentAccessIT Configuration (#7369)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new e5474f7d22 NIFI-11679 Refactored EncryptedRepoContentAccessIT
Configuration (#7369)
e5474f7d22 is described below
commit e5474f7d221644db7f2cac4d794344adbf486470
Author: exceptionfactory
AuthorDate: Tue Jun 13 10:06:45 2023 -0500
NIFI-11679 Refactored EncryptedRepoContentAccessIT Configuration (#7369)
---
.../repositories/EncryptedRepoContentAccessIT.java | 56 --
1 file changed, 52 insertions(+), 4 deletions(-)
diff --git
a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
index 2cfdb44473..6fb10f6fd9 100644
---
a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
+++
b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
@@ -17,17 +17,65 @@
package org.apache.nifi.tests.system.repositories;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.io.TempDir;
+
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.SecureRandom;
import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
public class EncryptedRepoContentAccessIT extends ContentAccessIT {
+private static final String KEYSTORE_CREDENTIALS =
UUID.randomUUID().toString();
+
+private static final String KEYSTORE_NAME = "repository.p12";
+
+private static final String KEY_ID = "primary-key";
+
+private static final String KEYSTORE_TYPE = "PKCS12";
+
+private static final int KEY_LENGTH = 32;
+
+private static final String KEY_ALGORITHM = "AES";
+
+private static Path keyStorePath;
+
+@BeforeAll
+public static void setRepositoryKeystore(@TempDir final Path
temporaryDirectory) throws GeneralSecurityException, IOException {
+keyStorePath = temporaryDirectory.resolve(KEYSTORE_NAME);
+
+final SecureRandom secureRandom = new SecureRandom();
+final byte[] key = new byte[KEY_LENGTH];
+secureRandom.nextBytes(key);
+final SecretKeySpec secretKeySpec = new SecretKeySpec(key,
KEY_ALGORITHM);
+
+final KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE);
+keyStore.load(null);
+
+final KeyStore.SecretKeyEntry secretKeyEntry = new
KeyStore.SecretKeyEntry(secretKeySpec);
+final KeyStore.PasswordProtection passwordProtection = new
KeyStore.PasswordProtection(KEYSTORE_CREDENTIALS.toCharArray());
+keyStore.setEntry(KEY_ID, secretKeyEntry, passwordProtection);
+
+try (final OutputStream outputStream =
Files.newOutputStream(keyStorePath)) {
+keyStore.store(outputStream, KEYSTORE_CREDENTIALS.toCharArray());
+}
+}
+
@Override
protected Map getNifiPropertiesOverrides() {
final Map encryptedRepoProperties = new HashMap<>();
-encryptedRepoProperties.put("nifi.content.repository.implementation",
"org.apache.nifi.controller.repository.crypto.EncryptedFileSystemRepository");
-encryptedRepoProperties.put("nifi.content.repository.encryption.key",
"0123456789ABCDEFFEDCBA9876543210");
-
encryptedRepoProperties.put("nifi.content.repository.encryption.key.id", "k1");
-
encryptedRepoProperties.put("nifi.content.repository.encryption.key.provider.implementation",
"StaticKeyProvider");
+
encryptedRepoProperties.put("nifi.repository.encryption.protocol.version", "1");
+encryptedRepoProperties.put("nifi.repository.encryption.key.id",
KEY_ID);
+encryptedRepoProperties.put("nifi.repository.encryption.key.provider",
"KEYSTORE");
+
encryptedRepoProperties.put("nifi.repository.encryption.key.provider.keystore.location",
keyStorePath.toString());
+
encryptedRepoProperties.put("nifi.repository.encryption.key.provider.keystore.password",
KEYSTORE_CREDENTIALS);
return encryptedRepoProperties;
}
}
[nifi] branch main updated: NIFI-11679 Refactored EncryptedRepoContentAccessIT Configuration (#7369)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 962dc9bc38 NIFI-11679 Refactored EncryptedRepoContentAccessIT
Configuration (#7369)
962dc9bc38 is described below
commit 962dc9bc388182d5163aeea7a8c0c2ceaaafaac5
Author: exceptionfactory
AuthorDate: Tue Jun 13 10:06:45 2023 -0500
NIFI-11679 Refactored EncryptedRepoContentAccessIT Configuration (#7369)
---
.../repositories/EncryptedRepoContentAccessIT.java | 56 --
1 file changed, 52 insertions(+), 4 deletions(-)
diff --git
a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
index 2cfdb44473..6fb10f6fd9 100644
---
a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
+++
b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/repositories/EncryptedRepoContentAccessIT.java
@@ -17,17 +17,65 @@
package org.apache.nifi.tests.system.repositories;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.io.TempDir;
+
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.SecureRandom;
import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
public class EncryptedRepoContentAccessIT extends ContentAccessIT {
+private static final String KEYSTORE_CREDENTIALS =
UUID.randomUUID().toString();
+
+private static final String KEYSTORE_NAME = "repository.p12";
+
+private static final String KEY_ID = "primary-key";
+
+private static final String KEYSTORE_TYPE = "PKCS12";
+
+private static final int KEY_LENGTH = 32;
+
+private static final String KEY_ALGORITHM = "AES";
+
+private static Path keyStorePath;
+
+@BeforeAll
+public static void setRepositoryKeystore(@TempDir final Path
temporaryDirectory) throws GeneralSecurityException, IOException {
+keyStorePath = temporaryDirectory.resolve(KEYSTORE_NAME);
+
+final SecureRandom secureRandom = new SecureRandom();
+final byte[] key = new byte[KEY_LENGTH];
+secureRandom.nextBytes(key);
+final SecretKeySpec secretKeySpec = new SecretKeySpec(key,
KEY_ALGORITHM);
+
+final KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE);
+keyStore.load(null);
+
+final KeyStore.SecretKeyEntry secretKeyEntry = new
KeyStore.SecretKeyEntry(secretKeySpec);
+final KeyStore.PasswordProtection passwordProtection = new
KeyStore.PasswordProtection(KEYSTORE_CREDENTIALS.toCharArray());
+keyStore.setEntry(KEY_ID, secretKeyEntry, passwordProtection);
+
+try (final OutputStream outputStream =
Files.newOutputStream(keyStorePath)) {
+keyStore.store(outputStream, KEYSTORE_CREDENTIALS.toCharArray());
+}
+}
+
@Override
protected Map getNifiPropertiesOverrides() {
final Map encryptedRepoProperties = new HashMap<>();
-encryptedRepoProperties.put("nifi.content.repository.implementation",
"org.apache.nifi.controller.repository.crypto.EncryptedFileSystemRepository");
-encryptedRepoProperties.put("nifi.content.repository.encryption.key",
"0123456789ABCDEFFEDCBA9876543210");
-
encryptedRepoProperties.put("nifi.content.repository.encryption.key.id", "k1");
-
encryptedRepoProperties.put("nifi.content.repository.encryption.key.provider.implementation",
"StaticKeyProvider");
+
encryptedRepoProperties.put("nifi.repository.encryption.protocol.version", "1");
+encryptedRepoProperties.put("nifi.repository.encryption.key.id",
KEY_ID);
+encryptedRepoProperties.put("nifi.repository.encryption.key.provider",
"KEYSTORE");
+
encryptedRepoProperties.put("nifi.repository.encryption.key.provider.keystore.location",
keyStorePath.toString());
+
encryptedRepoProperties.put("nifi.repository.encryption.key.provider.keystore.password",
KEYSTORE_CREDENTIALS);
return encryptedRepoProperties;
}
}
[nifi] branch support/nifi-1.x updated: NIFI-11680 Corrected Buffer Size Calculation for Connection Balancing (#7370)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 65436cb829 NIFI-11680 Corrected Buffer Size Calculation for Connection
Balancing (#7370)
65436cb829 is described below
commit 65436cb829472f1d2e9f4cb84ab87cf807c33825
Author: exceptionfactory
AuthorDate: Tue Jun 13 10:01:27 2023 -0500
NIFI-11680 Corrected Buffer Size Calculation for Connection Balancing
(#7370)
- Resolved BufferOverflowException in PeerChannel with Bouncy Castle
Provider
- Changed prepareForWrite() to use Destination Buffer remaining instead of
Application Buffer Size
- Changed encrypt() to Packet Buffer Size instead of Application Buffer Size
---
.../clustered/client/async/nio/PeerChannel.java | 20 +++-
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
index 5bee319089..abb69a3e87 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
@@ -162,12 +162,13 @@ public class PeerChannel implements Closeable {
while (plaintext.hasRemaining()) {
encrypt(plaintext);
-final int bytesRemaining = prepared.capacity() -
prepared.position();
-if (bytesRemaining < destinationBuffer.remaining()) {
-final ByteBuffer temp =
ByteBuffer.allocate(prepared.capacity() +
sslEngine.getSession().getApplicationBufferSize());
+final int destinationBufferRemaining =
destinationBuffer.remaining();
+if (prepared.remaining() < destinationBufferRemaining) {
+// Expand Prepared Buffer to hold current bytes plus remaining
size of Destination Buffer
+final ByteBuffer expanded =
ByteBuffer.allocate(prepared.capacity() + destinationBufferRemaining);
prepared.flip();
-temp.put(prepared);
-prepared = temp;
+expanded.put(prepared);
+prepared = expanded;
}
prepared.put(destinationBuffer);
@@ -289,11 +290,12 @@ public class PeerChannel implements Closeable {
case CLOSED:
throw new IOException("Failed to encrypt data to write to
Peer " + peerDescription + " because Peer unexpectedly closed connection");
case BUFFER_OVERFLOW:
-// destinationBuffer is not large enough. Need to increase
the size.
-final ByteBuffer tempBuffer =
ByteBuffer.allocate(destinationBuffer.capacity() +
sslEngine.getSession().getApplicationBufferSize());
+// Expand Destination Buffer using current capacity plus
encrypted Packet Buffer Size
+final int packetBufferSize =
sslEngine.getSession().getPacketBufferSize();
+final ByteBuffer expanded =
ByteBuffer.allocate(destinationBuffer.capacity() + packetBufferSize);
destinationBuffer.flip();
-tempBuffer.put(destinationBuffer);
-destinationBuffer = tempBuffer;
+expanded.put(destinationBuffer);
+destinationBuffer = expanded;
break;
case BUFFER_UNDERFLOW:
// We should never get this result on a call to
SSLEngine.wrap(), only on a call to unwrap().
[nifi] branch main updated: NIFI-11680 Corrected Buffer Size Calculation for Connection Balancing (#7370)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 9c2f15cc18 NIFI-11680 Corrected Buffer Size Calculation for Connection
Balancing (#7370)
9c2f15cc18 is described below
commit 9c2f15cc18a9fbb6e7035d3b70d5cc2036676fbc
Author: exceptionfactory
AuthorDate: Tue Jun 13 10:01:27 2023 -0500
NIFI-11680 Corrected Buffer Size Calculation for Connection Balancing
(#7370)
- Resolved BufferOverflowException in PeerChannel with Bouncy Castle
Provider
- Changed prepareForWrite() to use Destination Buffer remaining instead of
Application Buffer Size
- Changed encrypt() to Packet Buffer Size instead of Application Buffer Size
---
.../clustered/client/async/nio/PeerChannel.java | 20 +++-
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
index 5bee319089..abb69a3e87 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/client/async/nio/PeerChannel.java
@@ -162,12 +162,13 @@ public class PeerChannel implements Closeable {
while (plaintext.hasRemaining()) {
encrypt(plaintext);
-final int bytesRemaining = prepared.capacity() -
prepared.position();
-if (bytesRemaining < destinationBuffer.remaining()) {
-final ByteBuffer temp =
ByteBuffer.allocate(prepared.capacity() +
sslEngine.getSession().getApplicationBufferSize());
+final int destinationBufferRemaining =
destinationBuffer.remaining();
+if (prepared.remaining() < destinationBufferRemaining) {
+// Expand Prepared Buffer to hold current bytes plus remaining
size of Destination Buffer
+final ByteBuffer expanded =
ByteBuffer.allocate(prepared.capacity() + destinationBufferRemaining);
prepared.flip();
-temp.put(prepared);
-prepared = temp;
+expanded.put(prepared);
+prepared = expanded;
}
prepared.put(destinationBuffer);
@@ -289,11 +290,12 @@ public class PeerChannel implements Closeable {
case CLOSED:
throw new IOException("Failed to encrypt data to write to
Peer " + peerDescription + " because Peer unexpectedly closed connection");
case BUFFER_OVERFLOW:
-// destinationBuffer is not large enough. Need to increase
the size.
-final ByteBuffer tempBuffer =
ByteBuffer.allocate(destinationBuffer.capacity() +
sslEngine.getSession().getApplicationBufferSize());
+// Expand Destination Buffer using current capacity plus
encrypted Packet Buffer Size
+final int packetBufferSize =
sslEngine.getSession().getPacketBufferSize();
+final ByteBuffer expanded =
ByteBuffer.allocate(destinationBuffer.capacity() + packetBufferSize);
destinationBuffer.flip();
-tempBuffer.put(destinationBuffer);
-destinationBuffer = tempBuffer;
+expanded.put(destinationBuffer);
+destinationBuffer = expanded;
break;
case BUFFER_UNDERFLOW:
// We should never get this result on a call to
SSLEngine.wrap(), only on a call to unwrap().
[nifi] branch support/nifi-1.x updated: NIFI-11681: Terminate Process Sessions before interrupting processor threads
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new b85063cb81 NIFI-11681: Terminate Process Sessions before interrupting
processor threads
b85063cb81 is described below
commit b85063cb81210a733a214b7e2427d949ce51005f
Author: Mark Payne
AuthorDate: Mon Jun 12 16:55:41 2023 -0400
NIFI-11681: Terminate Process Sessions before interrupting processor threads
This closes #7371
Signed-off-by: David Handermann
(cherry picked from commit 787e0d82614e345b5afbef6691c418c43cf0ab08)
---
.../org/apache/nifi/controller/scheduling/StandardProcessScheduler.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
index e2aa0345b7..7c43431c2c 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
@@ -436,8 +436,8 @@ public final class StandardProcessScheduler implements
ProcessScheduler {
LOG.debug("Terminating {}", procNode);
-final int tasksTerminated = procNode.terminate();
state.terminate();
+final int tasksTerminated = procNode.terminate();
getSchedulingAgent(procNode).incrementMaxThreadCount(tasksTerminated);
[nifi] branch main updated: NIFI-11681: Terminate Process Sessions before interrupting processor threads
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 787e0d8261 NIFI-11681: Terminate Process Sessions before interrupting
processor threads
787e0d8261 is described below
commit 787e0d82614e345b5afbef6691c418c43cf0ab08
Author: Mark Payne
AuthorDate: Mon Jun 12 16:55:41 2023 -0400
NIFI-11681: Terminate Process Sessions before interrupting processor threads
This closes #7371
Signed-off-by: David Handermann
---
.../org/apache/nifi/controller/scheduling/StandardProcessScheduler.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
index c9633bf89d..b3e810c09b 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
@@ -436,8 +436,8 @@ public final class StandardProcessScheduler implements
ProcessScheduler {
LOG.debug("Terminating {}", procNode);
-final int tasksTerminated = procNode.terminate();
state.terminate();
+final int tasksTerminated = procNode.terminate();
getSchedulingAgent(procNode).incrementMaxThreadCount(tasksTerminated);
svn commit: r62399 - /release/nifi/KEYS
Author: kdoran Date: Tue Jun 13 13:00:11 2023 New Revision: 62399 Log: Adding committer’s code signing key to KEYS file Modified: release/nifi/KEYS Modified: release/nifi/KEYS == --- release/nifi/KEYS (original) +++ release/nifi/KEYS Tue Jun 13 13:00:11 2023 @@ -2062,3 +2062,62 @@ YGyDIBjSMKPCjz0dUtR1NNVMaeaSvjIyL2px+8DE q3SLZrvhGOPdiHwD5Uq+evcS1NYKXcFNLtcPNt4ngE74SPww0u/RFK4rzLzFTYo= =FDju -END PGP PUBLIC KEY BLOCK- +pub rsa4096 2023-03-17 [SC] + 2E7E B364 DE7B 4378 E49E 3B12 AFFF D8C3 A1A8 8ED7 +uid [ultimate] Nandor Soma Abonyi (CODE SIGNING KEY) +sig 3AFFFD8C3A1A88ED7 2023-03-17 [self-signature] +sub rsa4096 2023-03-17 [E] +sig AFFFD8C3A1A88ED7 2023-03-17 [self-signature] + +-BEGIN PGP PUBLIC KEY BLOCK- + +mQINBGQUgnkBEAC8vMdqLnqA05bZc9nR3PyEnW0wu0jPoZweOiYxRkeLriUav90w +NRT3VPISUuLEJ+KB1QpruScRY6YNBRl/4jRxts+5DWVkCX6jgWkWMnXzqv7rCwmr +oEjTom2TwgZqqJTYxOaVWaSu98gBUbtywM/3vluGHp5XdsS770HtDb9MLYtsiD6t +ZmBgKoUrYwjc8qnQ4ip0URiFxuLiKLnpWowqpcG2BwzXTkTPvYbJKjf2vGNNHbUS +wSddfSW/YYu9EceEa1iT6Fftam4xmdKsx2mIBP+XLiDhCP9B/CrJr4Tvh6bGEZlJ +pSv+/R11SmjVxIy/RGsw933jQRAcOwRt36vvYfhPWbXNn30iHU38LmwR4VNH22x3 +jTb4r+RYCUast3jzEeiMnISe1trkMZmcM2mwRFXmdilBHu7cpq9czBLacsk+s8uq +bI8yMSr4JQ08VjNv4/XuAUFq+aPUpZ1EgyjKqoUG58EcUQ0Jlrkyo5e2IT9Dy8ID +NkmyiQzxaMRcGFNITbfj2gr48mo16aODUCXl3ACDsbt3ArUhSQFqRlrlhBkJGqyq +H1EB2Q3ZeZBBkmbfldrQ5fVy4JOaEX+ukMJhL6AXCBecyg8HvRcCvCo27xcizWyS +S1GWaJyG/qYWLzUS0hvqPWX6UoYM2ZCwHdWCOPfMuKfEx/W5it5fqexTbQARAQAB +tDtOYW5kb3IgU29tYSBBYm9ueWkgKENPREUgU0lHTklORyBLRVkpIDxuc2Fib255 +aUBhcGFjaGUub3JnPokCUQQTAQgAOxYhBC5+s2Tee0N45J47Eq//2MOhqI7XBQJk +FIJ5AhsDBQsJCAcCAiICBhUKCQgLAgQWAgMBAh4HAheAAAoJEK//2MOhqI7XDIYP +/2i4766if6pyAaEzNvj1Az6ZX4+iQRi7npDjrPtjLohTSP3YtP3Kg/iXVKXRm6/g +QJ2KLeeFHZstrldbPNhA5YlONrOA1+45y3FPeq3sYzLnFZRYFRjUVv6xkVH5tYBQ +2HCCrgIn2vvSL8mdGhEm/1Bf47pwKJhFybJ0W8J7Cb7Yc1PNRG4SMC7Vc17wRbPB +GxETVcDpOjhc/c/sxqrjPTBkglM2GCgZcfLcHBIYBbc4y5076A4tO398eeHlVkc9 +YGyORXAqW29EwQE5RGexaK4QwqoTZErdWcbsR93BvpEql0QFzVPdTmhZgHBDcIoA +XJLWJD06yXuSgtVu+YQSb5jeT+8CakOouyv8OvZPZ8/TzGwQKTuYjVK0MSNuZl92 +XkuPGveaKx2KFeWsakY3vEaS5vhAlJOzUU1pCvx/5h9Lvyd8jiSdgy8QA8hvDciY +H1UUWZlEKxUOUvx14E47c12j5QjqFbIT+kO/lj9P1UM/NmgRo5ACr5tZFtCL0med +IFL20EItBPIM+Im97BYO2wlMZ6u2ecRIPxuG/5iEj2Sf2jXI5a1gdDOO9DHQEHVx +LNq8Zc/bHg5yjjdexRtDyMTZ0LnrsjPMT70mDQ/hx07h11WAtltp6B6CtLuAq7i5 +lRtAbqTto/GY7Z2qzyjEFtGZbCUlPxgKiSpTvy/rrM7JuQINBGQUgnkBEAC7V30D +z6Y8XLGZCJus+a8jXeegPN6bU49EgoOAE7YInnuAzj1o7ywUY2IKGS180bE9Vzpz +zLFjpHQzFOhHWDWzad2Vnx8I2JpRGaHLN7gBiQc8DDy8YUvGDvNZL0gFXQcTXSA+ +ncdo+oeLNgS+HXST0Fpm+9RCz4CSTtKkOB6y2Mw0HMd8G6j7DyOiM4biQjL61ug1 +LMzTMmMbGQzHcThSLd41PAKG6l44hg6eRF1Vz9A3zCeS6u0e7dxzvvuh+GklpWil +Xnofen66VwtpZKMdcbV25i1AD9zf+ZyKk9QhdzMujL4Htu4DiYFpLF0d8dpSyYn4 +vJhDMTNfvJmreSvpoEqR+cjXCA4ei15y4wBk6dAJ5BAb13lDyJCp1P6HlvIRXHlH +UvZN0MbaS5bSlflLaEcCtIQa+ySK04Sz/4cdYFUmR5xb2yndwdmMoA3NlghNwaSR +qT1bgbCTF0v8y7HwLX5JRS3QEuhZJMNyCrkGCjTBAddmi3vK2iH+ca9Hwq/GlLE0 +acOjg28nCTPsxuTwZA+ohdBsPNMQuTsvpwePzdOkzao5yQwxsVnI2frNJyqoQIY0 +aAkwBY1CPrYWH4QliDCW14i17puhj8q4sC172XAE3fWcTz8weObnF9FVQoq7wYFN +y/aeF6C8AV1AMjbZFDEPPA4Sb1a6i3NYHHCylQARAQABiQI2BBgBCAAgFiEELn6z +ZN57Q3jknjsSr//Yw6GojtcFAmQUgnkCGwwACgkQr//Yw6Gojtc4iA//Z5pTB+35 +HJddvvtGTZUAc53+zoAHeFgD3drmR9+TIVJ9AbwFDL++qStJx32o5wheCQ3DkknT +7uYJCi+SHmutzWYivp8SyskDJOgKGkLN2LYo2i8nq2806HpRRFjqp7gpP2k+APqb +OJgGXjN2bZ3iIkADSpViAM6xSZ50VfD9QsYo2plEazDQT94opi2WOl/aW0gOZTWY +q/YaBWBH3MnBWqcs/QZVnRtax97DRMgwXn1u4Bmy+aSNmtFRW0U0eOEp+Qqzb83a +sUnC/zy026WwESyRqjo9BavKBLGuzOoMtGfqnxCt5vDuwc6Gzau51UmSs9vVv6mi +w1Rcbp9c3Kg9gi7K00vxUz3FO8ZzdX0DJy6x2/XGlZTi3qaoB+FaS3RkLofYNYCk +I3xa6v92Ktjc3nmsl2uFRP6Bo+v1jTDR3d3QYhQfBOOx6bJ4C+H59Fn3SreD+ZT9 +NwYBw4Z5dT4nKWJuqpYch0PqccEvEDRr/i6GDhE756yzpkZ2GFvrO3VJxthihXk2 ++lEgN6pQ4K7ORnSnPOUk/A9gtWnA+LbqS1rzYAIKHxnlaBQnnI0GJKgnJQLuYpM3 +pgFJoG7sh/RJvROcKTNn1VwHFKwGoflN1sHn2jHWcD++JeMolYXhiLQdV29J4UzH +pKnZOa4ObDfC+OAR1d5cLXUZIrqxwna8skE= +=lpsK +-END PGP PUBLIC KEY BLOCK-
[nifi-minifi-cpp] 02/04: MINIFICPP-2081 - Make it possible to read Windows events from a log file
This is an automated email from the ASF dual-hosted git repository. fgerlits pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git commit 078fd1c87463ea227764f03b98b6bdc539074dfe Author: Adam Debreceni AuthorDate: Mon Apr 17 10:50:11 2023 +0200 MINIFICPP-2081 - Make it possible to read Windows events from a log file Signed-off-by: Ferenc Gerlits This closes #1564 --- PROCESSORS.md | 2 +- extensions/windows-event-log/Bookmark.cpp | 4 +- extensions/windows-event-log/Bookmark.h| 3 +- .../windows-event-log/ConsumeWindowsEventLog.cpp | 22 +--- .../windows-event-log/ConsumeWindowsEventLog.h | 6 +- .../windows-event-log/tests/BookmarkTests.cpp | 30 +- extensions/windows-event-log/tests/CWELTestUtils.h | 10 .../tests/ConsumeWindowsEventLogTests.cpp | 41 ++ extensions/windows-event-log/wel/EventPath.cpp | 39 + extensions/windows-event-log/wel/EventPath.h | 64 ++ 10 files changed, 204 insertions(+), 17 deletions(-) diff --git a/PROCESSORS.md b/PROCESSORS.md index 1fe86736b..c3ebccb10 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -460,7 +460,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values| Description [...] ||-|-|-- [...] -| **Channel**| System | | The Windows Event Log Channel to listen to.**Supports Expression Language: true** [...] +| **Channel**| System | | The Windows Event Log Channel to listen to. In order to process logs from a log file use the format 'SavedLog:\'. **Supports Expression Language: true** [...] | **Query** | * | | XPath Query to filter events. (See https://msdn.microsoft.com/en-us/library/windows/desktop/dd996910(v=vs.85).aspx for examples.)**Supports Expression Language: true** [...] | **Max Buffer Size**| 1 MB | | The individual Event Log XMLs are rendered to a buffer. This specifies the maximum size in bytes that the buffer will be allowed to grow to. (Limiting the maximum size of an individual Event XM [...] | **Inactive Duration To Reconnect** | 10 min | | If no new event logs are processed for the specified time period, this processor will try reconnecting to recover from a state where any further messages cannot be consumed. Such situation can [...] diff --git a/extensions/windows-event-log/Bookmark.cpp b/extensions/windows-event-log/Bookmark.cpp index
[nifi-minifi-cpp] branch main updated (9c3cdeca2 -> bbfa1e01b)
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
from 9c3cdeca2 MINIFICPP-2126 reenable Windows CI caching
new f4878bb70 MINIFICPP-1641 Parallelization of docker tests
new 078fd1c87 MINIFICPP-2081 - Make it possible to read Windows events
from a log file
new 66c958632 MINIFICPP-2128 - Verify the hash of a thirdparty dependency
new bbfa1e01b MINIFICPP-2132 improve error messages, refactor EL Value
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 111 ++-
.github/workflows/verify-python-compatibility.yml | 128 +++-
CMakeLists.txt | 2 +-
CMakeSettings.json | 4 +-
PROCESSORS.md | 2 +-
Windows.md | 2 +-
cmake/DockerConfig.cmake | 57 ++--
cmake/MiNiFiOptions.cmake | 3 +-
cmake/VerifyPythonCompatibility.cmake | 29 +-
docker/DockerVerify.sh | 57 ++--
docker/requirements.txt| 4 +-
docker/test/integration/cluster/ContainerStore.py | 224 --
.../test/integration/cluster/DockerCommunicator.py | 5 +-
.../test/integration/cluster/DockerTestCluster.py | 32 +-
.../cluster/DockerTestDirectoryBindings.py | 93 +++---
.../cluster/{LogSource.py => FeatureContext.py}| 8 +-
docker/test/integration/cluster/ImageStore.py | 41 ---
.../cluster/checkers/ElasticSearchChecker.py | 8 +-
.../containers/AzureStorageServerContainer.py | 4 +-
.../integration/cluster/containers/Container.py| 4 +-
.../cluster/containers/ElasticsearchContainer.py | 68 +++-
.../cluster/containers/FakeGcsServerContainer.py | 7 +-
.../cluster/containers/FlowContainer.py| 10 +-
.../cluster/containers/HttpProxyContainer.py | 5 +-
.../cluster/containers/KafkaBrokerContainer.py | 60 +++-
.../containers/MinifiAsPodInKubernetesCluster.py | 11 +-
.../cluster/containers/MinifiC2ServerContainer.py | 72 -
.../cluster/containers/MinifiContainer.py | 36 ++-
.../cluster/containers/MqttBrokerContainer.py | 5 +-
.../cluster/containers/NifiContainer.py| 4 +-
.../cluster/containers/OPCUAServerContainer.py | 5 +-
.../cluster/containers/OpensearchContainer.py | 50 ++-
.../containers/PostgreSQLServerContainer.py| 6 +-
.../cluster/containers/PrometheusContainer.py | 29 +-
.../cluster/containers/S3ServerContainer.py| 5 +-
.../cluster/containers/SplunkContainer.py | 4 +-
.../cluster/containers/SyslogTcpClientContainer.py | 7 +-
.../cluster/containers/SyslogUdpClientContainer.py | 7 +-
.../cluster/containers/TcpClientContainer.py | 10 +-
.../cluster/containers/ZookeeperContainer.py | 8 +-
docker/test/integration/features/CMakeLists.txt| 117 ---
.../MiNiFi_integration_test_driver.py | 67 ++--
docker/test/integration/{ => features}/README.md | 0
.../features/attributes_to_json.feature| 1 +
.../integration/features/azure_storage.feature | 1 +
.../features/core_functionality.feature| 8 +-
.../features/defragtextflowfiles.feature | 1 +
.../integration/features/elasticsearch.feature | 1 +
.../test/integration/{ => features}/environment.py | 61 ++--
.../features/file_system_operations.feature| 16 +
.../features/google_cloud_storage.feature | 1 +
.../test/integration/features/hashcontent.feature | 1 +
docker/test/integration/features/http.feature | 25 +-
docker/test/integration/features/https.feature | 3 +-
docker/test/integration/features/kafka.feature | 97 +++---
.../features/kubernetes_logging.feature| 1 +
.../features/kubernetes_metrics.feature| 1 +
.../test/integration/features/lua_script.feature | 1 +
.../integration/features/minifi_c2_server.feature | 1 +
.../integration/features/minifi_controller.feature | 1 +
docker/test/integration/features/mqtt.feature | 1 +
.../integration/features/network_listener.feature | 1 +
docker/test/integration/features/opcua.feature | 133
.../test/integration/features/opensearch.feature | 7 +-
.../test/integration/features/prometheus.feature | 1 +
docker/test/integration/features/python.feature| 1 +
.../integration/features/python_script.feature | 1 +
.../features/python_with_modules.feature | 2 +
[nifi-minifi-cpp] 04/04: MINIFICPP-2132 improve error messages, refactor EL Value
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit bbfa1e01b8d373700369d6cf356393b712e0
Author: Marton Szasz
AuthorDate: Wed Jun 7 13:32:40 2023 +0200
MINIFICPP-2132 improve error messages, refactor EL Value
Signed-off-by: Ferenc Gerlits
This closes #1585
---
extensions/expression-language/common/Value.h | 342 +++--
.../tests/ExpressionLanguageTests.cpp | 38 +++
.../processors/RouteOnAttribute.cpp| 14 +-
libminifi/include/utils/GeneralUtils.h | 8 +
4 files changed, 167 insertions(+), 235 deletions(-)
diff --git a/extensions/expression-language/common/Value.h
b/extensions/expression-language/common/Value.h
index 65d3cf586..ea524b77c 100644
--- a/extensions/expression-language/common/Value.h
+++ b/extensions/expression-language/common/Value.h
@@ -15,247 +15,141 @@
* limitations under the License.
*/
-#include
-#include
#include
#include
-#include
+#include
+#include
#include
+#include
+#include "utils/GeneralUtils.h"
+#include "utils/StringUtils.h"
#pragma once
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace expression {
-
+namespace org::apache::nifi::minifi::expression {
/**
- * Represents an expression value, which can be one of multiple types or NULL
+ * Represents an expression value, which can be one of multiple types or NULL
(represented as variant with monostate as null)
*/
class Value {
public:
- /**
- * Construct a default (NULL) value
- */
+ /// Construct a default (NULL) value
Value() = default;
- /**
- * Construct a string value
- */
- explicit Value(std::string val) {
-setString(std::move(val));
- }
-
- /**
- * Construct a boolean value
- */
- explicit Value(bool val) {
-setBoolean(val);
- }
-
- /**
- * Construct an unsigned long value
- */
- explicit Value(uint64_t val) {
-setUnsignedLong(val);
- }
-
- /**
- * Construct a signed long value
- */
- explicit Value(int64_t val) {
-setSignedLong(val);
- }
-
- /**
- * Construct a long double value
- */
- explicit Value(long double val) {
-setLongDouble(val);
- }
-
- bool isNull() const {
-return is_null_;
- }
-
- bool isString() const {
-return is_string_;
- }
-
- bool isDecimal() const {
-if (is_long_double_) {
- return true;
-} else if (is_string_ && (string_val_.find('.') != string_val_.npos ||
-string_val_.find('e') != string_val_.npos ||
-string_val_.find('E') != string_val_.npos)) {
- return true;
-} else {
- return false;
-}
- }
-
- void setSignedLong(int64_t val) {
-is_null_ = false;
-is_bool_ = false;
-is_signed_long_ = true;
-is_unsigned_long_ = false;
-is_long_double_ = false;
-is_string_ = false;
-signed_long_val_ = val;
- }
-
- void setUnsignedLong(uint64_t val) {
-is_null_ = false;
-is_bool_ = false;
-is_signed_long_ = false;
-is_unsigned_long_ = true;
-is_long_double_ = false;
-is_string_ = false;
-unsigned_long_val_ = val;
+ /// Construct with the specified value
+ explicit Value(std::string val) :value_{std::move(val)} {}
+ explicit Value(bool val) :value_{val} {}
+ explicit Value(uint64_t val) :value_{val} {}
+ explicit Value(int64_t val) :value_{val} {}
+ explicit Value(long double val) :value_{val} {}
+
+ [[nodiscard]] bool isNull() const { return
holds_alternative(value_); }
+
+ [[nodiscard]] bool isDecimal() const {
+return std::visit(utils::overloaded{
+ [](long double) { return true; },
+ [](const std::string& string_val) {
+return string_val.find('.') != string_val.npos ||
+string_val.find('e') != string_val.npos ||
+string_val.find('E') != string_val.npos;
+ },
+ [](const auto&) { return false; }
+}, value_);
+ }
+
+ void setSignedLong(int64_t val) { value_ = val; }
+ void setUnsignedLong(uint64_t val) { value_ = val; }
+ void setLongDouble(long double val) { value_ = val; }
+ void setBoolean(bool val) { value_ = val; }
+ void setString(std::string val) { value_ = std::move(val); }
+
+ [[nodiscard]] std::string asString() const {
+return std::visit(utils::overloaded{
+ [](const std::string& str) { return str; },
+ [](bool b) -> std::string { return b ? "true" : "false"; },
+ [](int64_t i) { return std::to_string(i); },
+ [](uint64_t i) { return std::to_string(i); },
+ [](long double d) {
+std::stringstream ss;
+ss << std::fixed <<
std::setprecision(std::numeric_limits::digits10)
+ << d;
+auto result = ss.str();
+result.erase(result.find_last_not_of('0') + 1, std::string::npos);
+
+if (result.find('.') == result.length() - 1) {
+ result.erase(result.length() - 1, std::string::npos);
[nifi-minifi-cpp] 03/04: MINIFICPP-2128 - Verify the hash of a thirdparty dependency
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 66c9586329582ab7f79f492eb118badda8a85dd9
Author: Adam Debreceni
AuthorDate: Wed May 31 13:55:49 2023 +0200
MINIFICPP-2128 - Verify the hash of a thirdparty dependency
Signed-off-by: Ferenc Gerlits
This closes #1582
---
.github/workflows/ci.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 11b9c618d..00c59f5c2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -96,6 +96,7 @@ jobs:
- id: install-sqliteodbc-driver
run: |
Invoke-WebRequest -Uri
"http://www.ch-werner.de/sqliteodbc/sqliteodbc_w64.exe; -OutFile
"sqliteodbc_w64.exe"
+ if ((Get-FileHash 'sqliteodbc_w64.exe').Hash -ne
"bd93e8450bcbd4ae16894e06ecb1b8572d6a01448df5b8dd2964eb6d3d302db7") {Write
"Hash mismatch"; Exit 1}
./sqliteodbc_w64.exe /S
shell: powershell
- name: build
