This is an automated email from the ASF dual-hosted git repository.
eolivelli pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new e73b393 [Functions] Completed work for PR #7555 to allow customizable
function logging (#10389)
e73b393 is described below
commit e73b393d633d147d2e7a836464e286c74d571aa7
Author: Devin Bost <[email protected]>
AuthorDate: Thu Apr 29 14:59:11 2021 -0600
[Functions] Completed work for PR #7555 to allow customizable function
logging (#10389)
* Completed work for PR #7555 to allow customizable function logging
* Fixed license header
* PR #10389 Added null check to prevent test failures.
* PR #10389 Added debug logging statement
* PR #10389 Made changes more robust, added default value, and fixed
NullPointerException
* PR #10389 Improved backwards compatibility
Co-authored-by: Devin Bost <[email protected]>
(cherry picked from commit d757db19c6553ab63674464310357b57fa906903)
---
bin/pulsar | 8 +-
conf/functions_log4j2.xml | 130 +++++++++++++++++++++
.../functions/runtime/process/ProcessRuntime.java | 12 +-
3 files changed, 148 insertions(+), 2 deletions(-)
diff --git a/bin/pulsar b/bin/pulsar
index 68566c2..1e8439e 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -31,6 +31,7 @@ DEFAULT_STANDALONE_CONF=$PULSAR_HOME/conf/standalone.conf
DEFAULT_WEBSOCKET_CONF=$PULSAR_HOME/conf/websocket.conf
DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
DEFAULT_PULSAR_PRESTO_CONF=${PULSAR_HOME}/conf/presto
+DEFAULT_FUNCTIONS_LOG_CONF=$PULSAR_HOME/conf/functions_log4j2.xml
# functions related variables
FUNCTIONS_HOME=$PULSAR_HOME/pulsar-functions
@@ -157,6 +158,7 @@ where command is one of:
or command is the full name of a class with a defined main() method.
Environment variables:
+ FUNCTIONS_LOG_CONF Function Log4j configuration file (default
$DEFAULT_FUNCTIONS_LOG_CONF)
PULSAR_LOG_CONF Log4j configuration file (default
$DEFAULT_LOG_CONF)
PULSAR_BROKER_CONF Configuration file for broker (default:
$DEFAULT_BROKER_CONF)
PULSAR_BOOKKEEPER_CONF Configuration file for bookie (default:
$DEFAULT_BOOKKEEPER_CONF)
@@ -264,6 +266,10 @@ if [ -z "$PULSAR_PRESTO_CONF" ]; then
PULSAR_PRESTO_CONF=$DEFAULT_PULSAR_PRESTO_CONF
fi
+if [ -z "$FUNCTIONS_LOG_CONF"]; then
+ FUNCTIONS_LOG_CONF=$DEFAULT_FUNCTIONS_LOG_CONF
+fi
+
PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
@@ -303,7 +309,7 @@ OPTS="$OPTS
-Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}"
OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}"
OPTS="$OPTS
-Dpulsar.functions.extra.dependencies.dir=${FUNCTIONS_EXTRA_DEPS_DIR}"
OPTS="$OPTS -Dpulsar.functions.instance.classpath=${PULSAR_CLASSPATH}"
-
+OPTS="$OPTS -Dpulsar.functions.log.conf=${FUNCTIONS_LOG_CONF}"
ZK_OPTS=" -Dzookeeper.4lw.commands.whitelist=*
-Dzookeeper.snapshot.trust.empty=true"
diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml
new file mode 100644
index 0000000..843ad9e
--- /dev/null
+++ b/conf/functions_log4j2.xml
@@ -0,0 +1,130 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<Configuration>
+ <name>pulsar-functions-instance</name>
+ <monitorInterval>30</monitorInterval>
+ <Properties>
+ <Property>
+ <name>pulsar.log.appender</name>
+ <value>RollingFile</value>
+ </Property>
+ <Property>
+ <name>pulsar.log.level</name>
+ <value>info</value>
+ </Property>
+ <Property>
+ <name>bk.log.level</name>
+ <value>info</value>
+ </Property>
+ </Properties>
+ <Appenders>
+ <Console>
+ <name>Console</name>
+ <target>SYSTEM_OUT</target>
+ <PatternLayout>
+ <Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} -
%msg%n</Pattern>
+ </PatternLayout>
+ </Console>
+ <RollingFile>
+ <name>RollingFile</name>
+
<fileName>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log</fileName>
+
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
+ <immediateFlush>true</immediateFlush>
+ <PatternLayout>
+ <Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} -
%msg%n</Pattern>
+ </PatternLayout>
+ <Policies>
+ <TimeBasedTriggeringPolicy>
+ <interval>1</interval>
+ <modulate>true</modulate>
+ </TimeBasedTriggeringPolicy>
+ <SizeBasedTriggeringPolicy>
+ <size>1 GB</size>
+ </SizeBasedTriggeringPolicy>
+ <CronTriggeringPolicy>
+ <schedule>0 0 0 * * ?</schedule>
+ </CronTriggeringPolicy>
+ </Policies>
+ <DefaultRolloverStrategy>
+ <Delete>
+ <basePath>${sys:pulsar.function.log.dir}</basePath>
+ <maxDepth>2</maxDepth>
+ <IfFileName>
+ <glob>*/${sys:pulsar.function.log.file}*log.gz</glob>
+ </IfFileName>
+ <IfLastModified>
+ <age>30d</age>
+ </IfLastModified>
+ </Delete>
+ </DefaultRolloverStrategy>
+ </RollingFile>
+ <RollingRandomAccessFile>
+ <name>BkRollingFile</name>
+
<fileName>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk</fileName>
+
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
+ <immediateFlush>true</immediateFlush>
+ <PatternLayout>
+ <Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} -
%msg%n</Pattern>
+ </PatternLayout>
+ <Policies>
+ <TimeBasedTriggeringPolicy>
+ <interval>1</interval>
+ <modulate>true</modulate>
+ </TimeBasedTriggeringPolicy>
+ <SizeBasedTriggeringPolicy>
+ <size>1 GB</size>
+ </SizeBasedTriggeringPolicy>
+ <CronTriggeringPolicy>
+ <schedule>0 0 0 * * ?</schedule>
+ </CronTriggeringPolicy>
+ </Policies>
+ <DefaultRolloverStrategy>
+ <Delete>
+ <basePath>${sys:pulsar.function.log.dir}</basePath>
+ <maxDepth>2</maxDepth>
+ <IfFileName>
+
<glob>*/${sys:pulsar.function.log.file}.bk*log.gz</glob>
+ </IfFileName>
+ <IfLastModified>
+ <age>30d</age>
+ </IfLastModified>
+ </Delete>
+ </DefaultRolloverStrategy>
+ </RollingRandomAccessFile>
+ </Appenders>
+ <Loggers>
+ <Logger>
+
<name>org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper</name>
+ <level>${sys:bk.log.level}</level>
+ <additivity>false</additivity>
+ <AppenderRef>
+ <ref>BkRollingFile</ref>
+ </AppenderRef>
+ </Logger>
+ <Root>
+ <level>info</level>
+ <AppenderRef>
+ <ref>${sys:pulsar.log.appender}</ref>
+ <level>${sys:pulsar.log.level}</level>
+ </AppenderRef>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java
index 0b75f3d..0b6d07b 100644
---
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java
+++
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java
@@ -42,6 +42,7 @@ import org.apache.pulsar.functions.runtime.RuntimeUtils;
import
org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator;
import org.apache.pulsar.functions.utils.FunctionCommon;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
@@ -102,7 +103,16 @@ class ProcessRuntime implements Runtime {
}
switch (instanceConfig.getFunctionDetails().getRuntime()) {
case JAVA:
- logConfigFile = "java_instance_log4j2.xml";
+ String logConfigPath =
System.getProperty("pulsar.functions.log.conf");
+ if(log.isDebugEnabled()){
+ log.debug("The loaded value of pulsar.functions.log.conf
is {}", logConfigPath);
+ }
+ // Added null check to prevent test failures
+ if(logConfigPath != null &&
Files.exists(Paths.get(logConfigPath))){
+ logConfigFile = logConfigPath;
+ } else { // Keeping existing file for backwards compatibility
+ logConfigFile = "java_instance_log4j2.xml";
+ }
break;
case PYTHON:
logConfigFile = System.getenv("PULSAR_HOME") +
"/conf/functions-logging/logging_config.ini";
