This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 97198e3 NIFI-9382: This closes #5584. Added system test that
replicates issue in which a closed shared classloader causes issues when used
again NIFI-9382: Fixed issue with SharedInstanceClassLoader where the
classloader may get closed but then get used again. When the
SharedInstanceClassLoader is closed, we will now ensure that we don't use
anymore and instead create a new one.
97198e3 is described below
commit 97198e35a04c12e66684d9545ff24156d16c60f6
Author: Mark Payne <[email protected]>
AuthorDate: Wed Dec 8 11:14:23 2021 -0500
NIFI-9382: This closes #5584. Added system test that replicates issue in
which a closed shared classloader causes issues when used again
NIFI-9382: Fixed issue with SharedInstanceClassLoader where the classloader
may get closed but then get used again. When the SharedInstanceClassLoader is
closed, we will now ensure that we don't use anymore and instead create a new
one.
Signed-off-by: Joe Witt <[email protected]>
---
.../processors/hadoop/AbstractHadoopProcessor.java | 2 +-
.../service/StandardControllerServiceNode.java | 5 +-
.../org/apache/nifi/nar/InstanceClassLoader.java | 4 --
.../nifi/nar/NarThreadContextClassLoader.java | 1 +
.../apache/nifi/nar/SharedInstanceClassLoader.java | 9 +++-
.../nar/StandardExtensionDiscoveringManager.java | 8 ++--
.../tests/system/WriteFlowFileCountToFile.java | 30 ++++++++++--
.../classloaders/ClassloaderIsolationKeyIT.java | 55 ++++++++++++++++++++++
.../resources/conf/clustered/node1/logback.xml | 41 +++++++++++++---
.../resources/conf/clustered/node2/logback.xml | 45 ++++++++++++++----
.../src/test/resources/conf/default/logback.xml | 43 +++++++++++++++--
11 files changed, 208 insertions(+), 35 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
b/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
index 222effd..7c574a7 100644
---
a/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
+++
b/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
@@ -205,7 +205,7 @@ public abstract class AbstractHadoopProcessor extends
AbstractProcessor implemen
@Override
public String getClassloaderIsolationKey(final PropertyContext context) {
- final String explicitKerberosPrincipal =
context.getProperty(kerberosProperties.getKerberosPrincipal()).getValue();
+ final String explicitKerberosPrincipal =
context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue();
if (explicitKerberosPrincipal != null) {
return explicitKerberosPrincipal;
}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
index bdae2b1..8593201 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
@@ -338,8 +338,9 @@ public class StandardControllerServiceNode extends
AbstractComponentNode impleme
@Override
public void verifyCanEnable() {
- if (getState() != ControllerServiceState.DISABLED) {
- throw new
IllegalStateException(getControllerServiceImplementation().getIdentifier() + "
cannot be enabled because it is not disabled");
+ final ControllerServiceState state = getState();
+ if (state != ControllerServiceState.DISABLED) {
+ throw new
IllegalStateException(getControllerServiceImplementation().getIdentifier() + "
cannot be enabled because it is not disabled - it has a state of " + state);
}
}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java
index e7153d7..b024bb5 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java
@@ -69,10 +69,6 @@ public class InstanceClassLoader extends
AbstractNativeLibHandlingClassLoader {
this.instanceType = type;
this.instanceUrls = instanceUrls == null ? Collections.emptySet() :
Collections.unmodifiableSet(new HashSet<>(instanceUrls));
this.additionalResourceUrls = additionalResourceUrls == null ?
Collections.emptySet() : Collections.unmodifiableSet(new
HashSet<>(additionalResourceUrls));
-
- if (parent instanceof SharedInstanceClassLoader) {
- ((SharedInstanceClassLoader) parent).incrementReferenceCount();
- }
}
@Override
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
index 949d207..915afa5 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
@@ -266,6 +266,7 @@ public class NarThreadContextClassLoader extends
URLClassLoader {
private static ClassLoader createClassLoader(final String
implementationClassName, final String instanceId, final Bundle bundle, final
ExtensionManager extensionManager)
throws ClassNotFoundException {
+
final ClassLoader bundleClassLoader = bundle.getClassLoader();
final Class<?> rawClass = Class.forName(implementationClassName, true,
bundleClassLoader);
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java
index 5aabef6..5ec1327 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java
@@ -24,6 +24,7 @@ import java.util.Set;
public class SharedInstanceClassLoader extends InstanceClassLoader {
private long referenceCount = 0L;
+ private boolean closed = false;
public SharedInstanceClassLoader(final String identifier, final String
type, final Set<URL> instanceUrls, final Set<URL> additionalResourceUrls,
final Set<File> narNativeLibDirs, final
ClassLoader parent) {
@@ -35,11 +36,17 @@ public class SharedInstanceClassLoader extends
InstanceClassLoader {
referenceCount--;
if (referenceCount <= 0) {
+ closed = true;
super.close();
}
}
- public synchronized void incrementReferenceCount() {
+ public synchronized boolean incrementReferenceCount() {
+ if (closed) {
+ return false;
+ }
+
referenceCount++;
+ return true;
}
}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java
index baa29b6..f8e1806 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java
@@ -88,7 +88,7 @@ public class StandardExtensionDiscoveringManager implements
ExtensionDiscovering
private final Map<String, ConfigurableComponent> tempComponentLookup = new
HashMap<>();
private final Map<String, InstanceClassLoader> instanceClassloaderLookup =
new ConcurrentHashMap<>();
- private final ConcurrentMap<BaseClassLoaderKey, ClassLoader>
sharedBaseClassloaders = new ConcurrentHashMap<>();
+ private final ConcurrentMap<BaseClassLoaderKey, SharedInstanceClassLoader>
sharedBaseClassloaders = new ConcurrentHashMap<>();
public StandardExtensionDiscoveringManager() {
this(Collections.emptyList());
@@ -407,8 +407,8 @@ public class StandardExtensionDiscoveringManager implements
ExtensionDiscovering
if (requiresInstanceClassLoading.cloneAncestorResources()) {
// Check to see if there's already a shared ClassLoader
that can be used as the parent/base classloader
if (baseClassLoaderKey != null) {
- final ClassLoader sharedBaseClassloader =
sharedBaseClassloaders.get(baseClassLoaderKey);
- if (sharedBaseClassloader != null) {
+ final SharedInstanceClassLoader sharedBaseClassloader
= sharedBaseClassloaders.get(baseClassLoaderKey);
+ if (sharedBaseClassloader != null &&
sharedBaseClassloader.incrementReferenceCount()) {
resolvedSharedClassLoader = true;
ancestorClassLoader = sharedBaseClassloader;
logger.debug("Creating InstanceClassLoader for
type {} using shared Base ClassLoader {} for component {}", type,
sharedBaseClassloader, instanceIdentifier);
@@ -444,6 +444,8 @@ public class StandardExtensionDiscoveringManager implements
ExtensionDiscovering
// Created a shared class loader that is everything we
need except for the additional URLs, as the additional URLs are
instance-specific.
final SharedInstanceClassLoader sharedClassLoader = new
SharedInstanceClassLoader(instanceIdentifier, classType, instanceUrls,
Collections.emptySet(), narNativeLibDirs,
ancestorClassLoader);
+ sharedClassLoader.incrementReferenceCount();
+
instanceClassLoader = new
InstanceClassLoader(instanceIdentifier, classType, Collections.emptySet(),
additionalUrls, Collections.emptySet(), sharedClassLoader);
logger.debug("Creating InstanceClassLoader for type {}
using newly created shared Base ClassLoader {} for component {}", type,
sharedClassLoader, instanceIdentifier);
diff --git
a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java
b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java
index 7e78f0a..ef6eb75 100644
---
a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java
+++
b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java
@@ -34,7 +34,7 @@ import java.io.FileOutputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
-import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
@@ -64,18 +64,31 @@ public class WriteFlowFileCountToFile extends
AbstractProcessor implements Class
.defaultValue("counts.txt")
.build();
+ static final PropertyDescriptor CLASS_TO_CREATE = new Builder()
+ .name("Class to Create")
+ .displayName("Class to Create")
+ .description("If specified, each iteration of #onTrigger will create
an instance of this class in order to test ClassLoader behavior. If unable to
create the object, the FlowFile will be " +
+ "routed to failure")
+ .required(false)
+ .addValidator(NON_EMPTY_VALIDATOR)
+ .build();
+
private final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.build();
+ private final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .autoTerminateDefault(true)
+ .build();
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
- return Arrays.asList(ISOLATION_KEY, FILE_TO_WRITE);
+ return Arrays.asList(ISOLATION_KEY, FILE_TO_WRITE, CLASS_TO_CREATE);
}
@Override
public Set<Relationship> getRelationships() {
- return Collections.singleton(REL_SUCCESS);
+ return new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE));
}
@Override
@@ -90,6 +103,17 @@ public class WriteFlowFileCountToFile extends
AbstractProcessor implements Class
return;
}
+ final String className =
context.getProperty(CLASS_TO_CREATE).getValue();
+ if (className != null) {
+ try {
+ Class.forName(className, true,
Thread.currentThread().getContextClassLoader());
+ } catch (final ClassNotFoundException e) {
+ getLogger().error("Failed to load class {} for {}; routing to
failure", className, flowFile);
+ session.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+ }
+
final long counterValue = counter.incrementAndGet();
final byte[] fileContents =
String.valueOf(counterValue).getBytes(StandardCharsets.UTF_8);
diff --git
a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java
b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java
index 0665ae8..4c5fb82 100644
---
a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java
+++
b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java
@@ -19,6 +19,7 @@ package org.apache.nifi.tests.system.classloaders;
import org.apache.nifi.tests.system.NiFiSystemIT;
import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException;
+import org.apache.nifi.web.api.entity.ConnectionEntity;
import org.apache.nifi.web.api.entity.ProcessorEntity;
import org.junit.Test;
@@ -28,8 +29,62 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Collections;
+import static org.junit.Assert.assertEquals;
+
public class ClassloaderIsolationKeyIT extends NiFiSystemIT {
+ /**
+ * After creating 1+ processors with the same ClassLoader Isolation Key,
and then removing them,
+ * the SharedInstanceClassLoader will be closed. If we then create a new
processor with the same
+ * ClassLoader Isolation Key, we need to ensure that we are then able to
load classes from the ClassLoader
+ * that were not loaded previously.
+ */
+ @Test
+ public void
testRemoveAllInstancesThenCreateForSameIsolationKeyAllowsClassLoading() throws
NiFiClientException, IOException, InterruptedException {
+ final ProcessorEntity generate =
getClientUtil().createProcessor("GenerateFlowFile");
+ final ProcessorEntity counter =
getClientUtil().createProcessor("WriteFlowFileCountToFile");
+ final ProcessorEntity terminate =
getClientUtil().createProcessor("TerminateFlowFile");
+
+ getClientUtil().updateProcessorProperties(counter,
Collections.singletonMap("File to Write", "count1.txt"));
+ getClientUtil().updateProcessorProperties(counter,
Collections.singletonMap("Isolation Key", "abc123"));
+
+ getClientUtil().createConnection(generate, counter, "success");
+ final ConnectionEntity counterToTerminate =
getClientUtil().createConnection(counter, terminate, "success");
+
+ getClientUtil().waitForValidProcessor(counter.getId());
+
+ getClientUtil().startProcessor(generate);
+ getClientUtil().startProcessor(counter);
+
+ waitForQueueCount(counterToTerminate.getId(), 1);
+
+ // Stop components, purge FlowFiles, delete all components
+ destroyFlow();
+
+ final ProcessorEntity newGenerate =
getClientUtil().createProcessor("GenerateFlowFile");
+ final ProcessorEntity newCounter =
getClientUtil().createProcessor("WriteFlowFileCountToFile");
+ final ProcessorEntity terminateSuccess =
getClientUtil().createProcessor("TerminateFlowFile");
+ final ProcessorEntity terminateFailure =
getClientUtil().createProcessor("TerminateFlowFile");
+
+ final ConnectionEntity generateToCounter =
getClientUtil().createConnection(newGenerate, newCounter, "success");
+ final ConnectionEntity counterSuccess =
getClientUtil().createConnection(newCounter, terminateSuccess, "success");
+ final ConnectionEntity counterFailure =
getClientUtil().createConnection(newCounter, terminateFailure, "failure");
+
+ getClientUtil().updateProcessorProperties(newCounter,
Collections.singletonMap("Class to Create",
"org.apache.nifi.processors.tests.system.CountEvents"));
+ getClientUtil().updateProcessorProperties(newCounter,
Collections.singletonMap("File to Write", "count1.txt"));
+ getClientUtil().updateProcessorProperties(newCounter,
Collections.singletonMap("Isolation Key", "abc123"));
+
+ getClientUtil().waitForValidProcessor(newCounter.getId());
+
+ getClientUtil().startProcessor(newGenerate);
+ getClientUtil().startProcessor(newCounter);
+
+ waitForQueueCount(generateToCounter.getId(), 0);
+ assertEquals(0, getConnectionQueueSize(counterFailure.getId()));
+ assertEquals(1, getConnectionQueueSize(counterSuccess.getId()));
+ }
+
+
@Test
public void testClassloaderChanges() throws NiFiClientException,
IOException, InterruptedException {
final ProcessorEntity generate =
getClientUtil().createProcessor("GenerateFlowFile");
diff --git
a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml
b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml
index 6c3f0bb..c9bc033 100644
---
a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml
+++
b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml
@@ -57,6 +57,17 @@
</encoder>
</appender>
+ <appender name="REQUEST_FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
+ <rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%msg%n</pattern>
+ </encoder>
+ </appender>
+
<appender name="BOOTSTRAP_FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
@@ -82,21 +93,22 @@
</appender>
<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
+
<logger name="org.apache.nifi" level="INFO"/>
<logger name="org.apache.nifi.processors" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogAttribute"
level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogMessage"
level="INFO"/>
<logger
name="org.apache.nifi.controller.repository.StandardProcessSession"
level="WARN" />
- <logger name="org.apache.nifi.connectable.LocalPort" level="DEBUG"/>
- <logger
name="org.apache.nifi.web.util.ClusterReplicationComponentLifecycle"
level="DEBUG" />
<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxnFactory"
level="ERROR" />
+ <logger name="org.apache.zookeeper.server.NettyServerCnxnFactory"
level="ERROR" />
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
<logger name="org.apache.zookeeper.server.PrepRequestProcessor"
level="ERROR" />
+ <logger name="org.apache.nifi.controller.reporting.LogComponentStatuses"
level="ERROR" />
<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />
@@ -128,6 +140,17 @@
<!-- Suppress non-error messages from SMBJ which was emitting large
amounts of INFO logs by default -->
<logger name="com.hierynomus.smbj" level="WARN" />
+ <!-- Suppress non-error messages from AWS KCL which was emitting large
amounts of INFO logs by default -->
+ <logger name="com.amazonaws.services.kinesis" level="WARN" />
+
+ <!-- Suppress non-error messages from Apache Atlas which was emitting
large amounts of INFO logs by default -->
+ <logger name="org.apache.atlas" level="WARN"/>
+
+ <!-- These log messages would normally go to the USER_FILE log, but they
belong in the APP_FILE -->
+ <logger name="org.apache.nifi.web.security.requests" level="INFO"
additivity="false">
+ <appender-ref ref="APP_FILE"/>
+ </logger>
+
<!--
Logger for capturing user events. We do not want to propagate these
log events to the root logger. These messages are only sent to the
@@ -148,10 +171,17 @@
<logger name="org.apache.nifi.web.api.AccessResource" level="INFO"
additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
- <logger name="org.apache.nifi.web.api" level="DEBUG" additivity="false">
+ <logger name="org.springframework.security.saml.log" level="WARN"
additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <logger name="org.opensaml" level="WARN" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
+ <!-- Web Server Request Log -->
+ <logger name="org.apache.nifi.web.server.RequestLog" level="INFO"
additivity="false">
+ <appender-ref ref="REQUEST_FILE"/>
+ </logger>
<!--
Logger for capturing Bootstrap logs and NiFi's standard error and
standard out.
@@ -174,9 +204,8 @@
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
-
<root level="INFO">
- <appender-ref ref="APP_FILE"/>
+ <appender-ref ref="APP_FILE" />
</root>
-</configuration>
\ No newline at end of file
+</configuration>
diff --git
a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml
b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml
index a9cdb26..c9bc033 100644
---
a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml
+++
b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml
@@ -57,6 +57,17 @@
</encoder>
</appender>
+ <appender name="REQUEST_FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
+ <rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%msg%n</pattern>
+ </encoder>
+ </appender>
+
<appender name="BOOTSTRAP_FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
@@ -82,25 +93,22 @@
</appender>
<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
+
<logger name="org.apache.nifi" level="INFO"/>
<logger name="org.apache.nifi.processors" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogAttribute"
level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogMessage"
level="INFO"/>
<logger
name="org.apache.nifi.controller.repository.StandardProcessSession"
level="WARN" />
- <logger name="org.apache.nifi.connectable.LocalPort" level="DEBUG"/>
- <logger
name="org.apache.nifi.web.util.ClusterReplicationComponentLifecycle"
level="DEBUG" />
- <logger name="org.apache.nifi.groups.AbstractComponentScheduler"
level="DEBUG" />
- <logger name="org.apache.nifi.controller.XmlFlowSynchronizer"
level="DEBUG" />
- <logger name="org.apache.nifi.controller.inheritance" level="DEBUG" />
- <logger
name="org.apache.nifi.controller.serialization.VersionedFlowSynchronizer"
level="DEBUG" />
<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxnFactory"
level="ERROR" />
+ <logger name="org.apache.zookeeper.server.NettyServerCnxnFactory"
level="ERROR" />
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
<logger name="org.apache.zookeeper.server.PrepRequestProcessor"
level="ERROR" />
+ <logger name="org.apache.nifi.controller.reporting.LogComponentStatuses"
level="ERROR" />
<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />
@@ -132,6 +140,17 @@
<!-- Suppress non-error messages from SMBJ which was emitting large
amounts of INFO logs by default -->
<logger name="com.hierynomus.smbj" level="WARN" />
+ <!-- Suppress non-error messages from AWS KCL which was emitting large
amounts of INFO logs by default -->
+ <logger name="com.amazonaws.services.kinesis" level="WARN" />
+
+ <!-- Suppress non-error messages from Apache Atlas which was emitting
large amounts of INFO logs by default -->
+ <logger name="org.apache.atlas" level="WARN"/>
+
+ <!-- These log messages would normally go to the USER_FILE log, but they
belong in the APP_FILE -->
+ <logger name="org.apache.nifi.web.security.requests" level="INFO"
additivity="false">
+ <appender-ref ref="APP_FILE"/>
+ </logger>
+
<!--
Logger for capturing user events. We do not want to propagate these
log events to the root logger. These messages are only sent to the
@@ -152,10 +171,17 @@
<logger name="org.apache.nifi.web.api.AccessResource" level="INFO"
additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
- <logger name="org.apache.nifi.web.api" level="DEBUG" additivity="false">
+ <logger name="org.springframework.security.saml.log" level="WARN"
additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <logger name="org.opensaml" level="WARN" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
+ <!-- Web Server Request Log -->
+ <logger name="org.apache.nifi.web.server.RequestLog" level="INFO"
additivity="false">
+ <appender-ref ref="REQUEST_FILE"/>
+ </logger>
<!--
Logger for capturing Bootstrap logs and NiFi's standard error and
standard out.
@@ -178,9 +204,8 @@
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
-
<root level="INFO">
- <appender-ref ref="APP_FILE"/>
+ <appender-ref ref="APP_FILE" />
</root>
-</configuration>
\ No newline at end of file
+</configuration>
diff --git
a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml
b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml
index c42b3be..c9bc033 100644
---
a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml
+++
b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml
@@ -57,6 +57,17 @@
</encoder>
</appender>
+ <appender name="REQUEST_FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
+ <rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%msg%n</pattern>
+ </encoder>
+ </appender>
+
<appender name="BOOTSTRAP_FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
@@ -82,20 +93,22 @@
</appender>
<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
- <logger name="org.apache.nifi.web.util.LocalComponentLifecycle"
level="DEBUG" />
+
<logger name="org.apache.nifi" level="INFO"/>
<logger name="org.apache.nifi.processors" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogAttribute"
level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogMessage"
level="INFO"/>
<logger
name="org.apache.nifi.controller.repository.StandardProcessSession"
level="WARN" />
- <logger name="org.apache.nifi.connectable.LocalPort" level="DEBUG"/>
+
<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxnFactory"
level="ERROR" />
+ <logger name="org.apache.zookeeper.server.NettyServerCnxnFactory"
level="ERROR" />
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
<logger name="org.apache.zookeeper.server.PrepRequestProcessor"
level="ERROR" />
+ <logger name="org.apache.nifi.controller.reporting.LogComponentStatuses"
level="ERROR" />
<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />
@@ -127,6 +140,17 @@
<!-- Suppress non-error messages from SMBJ which was emitting large
amounts of INFO logs by default -->
<logger name="com.hierynomus.smbj" level="WARN" />
+ <!-- Suppress non-error messages from AWS KCL which was emitting large
amounts of INFO logs by default -->
+ <logger name="com.amazonaws.services.kinesis" level="WARN" />
+
+ <!-- Suppress non-error messages from Apache Atlas which was emitting
large amounts of INFO logs by default -->
+ <logger name="org.apache.atlas" level="WARN"/>
+
+ <!-- These log messages would normally go to the USER_FILE log, but they
belong in the APP_FILE -->
+ <logger name="org.apache.nifi.web.security.requests" level="INFO"
additivity="false">
+ <appender-ref ref="APP_FILE"/>
+ </logger>
+
<!--
Logger for capturing user events. We do not want to propagate these
log events to the root logger. These messages are only sent to the
@@ -147,7 +171,17 @@
<logger name="org.apache.nifi.web.api.AccessResource" level="INFO"
additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
+ <logger name="org.springframework.security.saml.log" level="WARN"
additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <logger name="org.opensaml" level="WARN" additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+ <!-- Web Server Request Log -->
+ <logger name="org.apache.nifi.web.server.RequestLog" level="INFO"
additivity="false">
+ <appender-ref ref="REQUEST_FILE"/>
+ </logger>
<!--
Logger for capturing Bootstrap logs and NiFi's standard error and
standard out.
@@ -170,9 +204,8 @@
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
-
<root level="INFO">
- <appender-ref ref="APP_FILE"/>
+ <appender-ref ref="APP_FILE" />
</root>
-</configuration>
\ No newline at end of file
+</configuration>
