[nifi-registry] branch master updated: NIFIREG-285 - Add DatabaseFlowPersistenceProvider
This is an automated email from the ASF dual-hosted git repository. kdoran pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nifi-registry.git The following commit(s) were added to refs/heads/master by this push: new 6f19a73 NIFIREG-285 - Add DatabaseFlowPersistenceProvider 6f19a73 is described below commit 6f19a736c00242affb82a0d326e686be5a2e1b50 Author: Bryan Bende AuthorDate: Mon Jul 15 11:49:21 2019 -0400 NIFIREG-285 - Add DatabaseFlowPersistenceProvider - Added method injection of DataSource in StandardProviderFactory - Split out postgres migrations to use BYTEA for blob type This closes #205. Signed-off-by: Kevin Doran --- .../src/main/asciidoc/administration-guide.adoc| 23 ++- .../registry/db/CustomFlywayConfiguration.java | 7 + .../registry/provider/StandardProviderFactory.java | 52 +- .../flow/DatabaseFlowPersistenceProvider.java | 86 ++ ...ache.nifi.registry.flow.FlowPersistenceProvider | 3 +- .../migration/default/V6__AddFlowPersistence.sql | 22 +++ .../db/migration/mysql/V6__AddFlowPersistence.sql | 22 +++ .../db/migration/postgres/V2__Initial.sql | 60 +++ .../db/migration/postgres/V3__AddExtensions.sql| 105 .../migration/postgres/V4__AddCascadeOnDelete.sql | 23 +++ .../postgres/V5__AddBucketPublicFlags.sql | 16 ++ .../migration/postgres/V6__AddFlowPersistence.sql | 22 +++ .../provider/TestStandardProviderFactory.java | 18 ++- .../flow/TestDatabaseFlowPersistenceProvider.java | 100 .../provider/hook/TestScriptEventHookProvider.java | 12 +- .../nifi/registry/provider/ProviderContext.java| 34 .../src/main/resources/conf/providers.xml | 6 + .../nifi/registry/web/api/DBFlowStorageIT.java | 178 + .../application-ITDBFlowStorage.properties}| 9 +- .../conf/db-flow-storage/nifi-registry.properties} | 16 +- .../resources/conf/providers-db-flow-storage.xml | 29 .../FlowPersistenceProviderMigrator.java | 13 +- 22 files changed, 832 insertions(+), 24 deletions(-) diff --git a/nifi-registry-core/nifi-registry-docs/src/main/asciidoc/administration-guide.adoc b/nifi-registry-core/nifi-registry-docs/src/main/asciidoc/administration-guide.adoc index 29b5589..a28dc50 100644 --- a/nifi-registry-core/nifi-registry-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-registry-core/nifi-registry-docs/src/main/asciidoc/administration-guide.adoc @@ -1118,7 +1118,7 @@ Currently, NiFi Registry supports using H2, Postgres 9.x, and MySQL (5.6, 5.7, 8 NOTE: NiFi Registry 0.1.0 only supports H2. -== H2 +=== H2 H2 is an embedded database that is pre-configured in the default _nifi-registry.properties_ file. The contents of the H2 database are stored in a file on the local filesystem. @@ -1130,7 +1130,7 @@ For NiFi Registry 0.2.0 and forward, the location of the H2 database is specifie `nifi.registry.db.url=jdbc:h2:./database/nifi-registry-primary;` -== Postgres +=== Postgres Postgres provides the option to use an externally located database that also supports high availability. @@ -1159,7 +1159,7 @@ The following steps are required to use Postgres: nifi.registry.db.username=nifireg nifi.registry.db.password=changeme -== MySQL +=== MySQL MySQL also provides the option to use an externally located database that also supports high availability. @@ -1357,6 +1357,23 @@ Host bitbucket.org IdentityFile ~/.ssh/key-for-bitbucket + DatabaseFlowPersistenceProvider + +`DatabaseFlowPersistenceProvider` stores flow contents in a database table. + +This provider leverages the same database used for the metadata database, so there is no configuration to provide since the +connection details will come from the database properties in `nifi-registry.properties`. + +The database table is named `FLOW_PERSISTENCE_PROVIDER` and has the following schema: + +| +|*Column*|*Description* +|BUCKET_ID|The identifier of the bucket where the flow is located. +|FLOW_ID|The identifier of the flow. +|VERSION|The version of the flow. +|FLOW_CONTENT|The serialized bytes of the flow content stored as a BLOB. +| + Switching from other Flow Persistence Provider In order to switch the Flow Persistence Provider, it is necessary to reset NiFi Registry. diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/CustomFlywayConfiguration.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/CustomFlywayConfiguration.java index 0cbf64f..0288f9d 100644 --- a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/CustomFlywayConfiguration.java +++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/CustomFlywayConfiguration.java @@ -41,6 +41,9 @@ public class
[nifi] branch master updated: NIFI-6507 Unsubscribe if WindowsEventLog subscription encounters an error
This is an automated email from the ASF dual-hosted git repository.
pvillard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new 05cbf8b NIFI-6507 Unsubscribe if WindowsEventLog subscription
encounters an error
05cbf8b is described below
commit 05cbf8b24ff6b17c69cc12311d7a786c295d75b4
Author: Koji Kawamura
AuthorDate: Mon Jul 1 11:39:20 2019 +0900
NIFI-6507 Unsubscribe if WindowsEventLog subscription encounters an error
Signed-off-by: Pierre Villard
This closes #3617.
---
.../windows/event/log/ConsumeWindowsEventLog.java | 11 ++-
.../jna/EventSubscribeXmlRenderingCallback.java| 22 ++
2 files changed, 28 insertions(+), 5 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-windows-event-log-bundle/nifi-windows-event-log-processors/src/main/java/org/apache/nifi/processors/windows/event/log/ConsumeWindowsEventLog.java
b/nifi-nar-bundles/nifi-windows-event-log-bundle/nifi-windows-event-log-processors/src/main/java/org/apache/nifi/processors/windows/event/log/ConsumeWindowsEventLog.java
index 7dc4236..22b2b34 100644
---
a/nifi-nar-bundles/nifi-windows-event-log-bundle/nifi-windows-event-log-processors/src/main/java/org/apache/nifi/processors/windows/event/log/ConsumeWindowsEventLog.java
+++
b/nifi-nar-bundles/nifi-windows-event-log-bundle/nifi-windows-event-log-processors/src/main/java/org/apache/nifi/processors/windows/event/log/ConsumeWindowsEventLog.java
@@ -240,7 +240,16 @@ public class ConsumeWindowsEventLog extends
AbstractSessionFactoryProcessor {
}
private boolean isSubscribed() {
-return subscriptionHandle != null && subscriptionHandle.getPointer()
!= null;
+final boolean subscribed = subscriptionHandle != null &&
subscriptionHandle.getPointer() != null;
+final boolean subscriptionFailed = evtSubscribeCallback != null
+&& ((EventSubscribeXmlRenderingCallback)
evtSubscribeCallback).isSubscriptionFailed();
+final boolean subscribing = subscribed && !subscriptionFailed;
+getLogger().debug("subscribing? {}, subscribed={},
subscriptionFailed={}", new Object[]{subscribing, subscribed,
subscriptionFailed});
+if (subscriptionFailed) {
+getLogger().info("Canceling a failed subscription.");
+unsubscribe();
+}
+return subscribing;
}
@OnScheduled
diff --git
a/nifi-nar-bundles/nifi-windows-event-log-bundle/nifi-windows-event-log-processors/src/main/java/org/apache/nifi/processors/windows/event/log/jna/EventSubscribeXmlRenderingCallback.java
b/nifi-nar-bundles/nifi-windows-event-log-bundle/nifi-windows-event-log-processors/src/main/java/org/apache/nifi/processors/windows/event/log/jna/EventSubscribeXmlRenderingCallback.java
index 451e446..82c94ee 100644
---
a/nifi-nar-bundles/nifi-windows-event-log-bundle/nifi-windows-event-log-processors/src/main/java/org/apache/nifi/processors/windows/event/log/jna/EventSubscribeXmlRenderingCallback.java
+++
b/nifi-nar-bundles/nifi-windows-event-log-bundle/nifi-windows-event-log-processors/src/main/java/org/apache/nifi/processors/windows/event/log/jna/EventSubscribeXmlRenderingCallback.java
@@ -46,6 +46,7 @@ public class EventSubscribeXmlRenderingCallback implements
WEvtApi.EVT_SUBSCRIBE
private Memory buffer;
private Memory used;
private Memory propertyCount;
+private boolean subscriptionFailed;
public EventSubscribeXmlRenderingCallback(ComponentLog logger,
Consumer consumer, int maxBufferSize, WEvtApi wEvtApi, Kernel32
kernel32, ErrorLookup errorLookup) {
this.logger = logger;
@@ -67,11 +68,20 @@ public class EventSubscribeXmlRenderingCallback implements
WEvtApi.EVT_SUBSCRIBE
}
if (evtSubscribeNotifyAction ==
WEvtApi.EvtSubscribeNotifyAction.ERROR) {
-if (eventHandle.getPointer().getInt(0) ==
WEvtApi.EvtSubscribeErrors.ERROR_EVT_QUERY_RESULT_STALE) {
-logger.error(MISSING_EVENT_MESSAGE);
-} else {
-logger.error(RECEIVED_THE_FOLLOWING_WIN32_ERROR +
eventHandle.getPointer().getInt(0));
+try {
+final int errorCode = eventHandle.getPointer().getInt(0);
+if (errorCode ==
WEvtApi.EvtSubscribeErrors.ERROR_EVT_QUERY_RESULT_STALE) {
+logger.error(MISSING_EVENT_MESSAGE);
+} else {
+logger.error(RECEIVED_THE_FOLLOWING_WIN32_ERROR +
errorCode);
+}
+} catch (final Error e) {
+logger.error("Failed to get error code onEvent("
++ evtSubscribeNotifyAction + ", " + userContext + ", " +
eventHandle);
+} finally {
+subscriptionFailed = true;
}
+
} else if (evtSubscribeNotifyAction ==
WEvtApi.EvtSubscribeNotifyAction.DELIVER) {
