This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 3428d49 fixing and standardizing logging in functions (#1556)
3428d49 is described below
commit 3428d498e3438a24defa279f45648b3be32b38a4
Author: Boyang Jerry Peng <jerry.boyang.p...@gmail.com>
AuthorDate: Wed Apr 11 22:03:27 2018 -0700
fixing and standardizing logging in functions (#1556)
* fixing and standardizing logging in functions
* fix unit tests
---
pulsar-functions/instance/src/main/python/log.py | 9 +++++++++
.../instance/src/main/python/python_instance_main.py | 5 ++++-
pulsar-functions/instance/src/main/python/util.py | 3 +++
.../apache/pulsar/functions/runtime/ProcessRuntimeFactory.java | 1 +
.../runtime/src/main/resources/java_instance_log4j2.yml | 6 +++---
.../org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java | 4 ++--
6 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/pulsar-functions/instance/src/main/python/log.py
b/pulsar-functions/instance/src/main/python/log.py
index 18bde78..c1a1420 100644
--- a/pulsar-functions/instance/src/main/python/log.py
+++ b/pulsar-functions/instance/src/main/python/log.py
@@ -22,6 +22,8 @@
''' log.py '''
import logging
+import os
+import errno
from logging.handlers import RotatingFileHandler
import pulsar
@@ -88,6 +90,13 @@ def init_rotating_logger(level, logfile, max_files,
max_bytes):
It also makes sure that any StreamHandler is removed, so as to avoid
stdout/stderr
constipation issues
"""
+ # create log directory if necessary
+ try:
+ os.makedirs(os.path.dirname(logfile))
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
logging.basicConfig()
root_logger = logging.getLogger()
diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py
b/pulsar-functions/instance/src/main/python/python_instance_main.py
index 87ba549..2638fa5 100644
--- a/pulsar-functions/instance/src/main/python/python_instance_main.py
+++ b/pulsar-functions/instance/src/main/python/python_instance_main.py
@@ -36,6 +36,7 @@ import Function_pb2
import log
import server
import python_instance
+import util
to_run = True
Log = log.Log
@@ -76,7 +77,9 @@ def main():
parser.add_argument('--log_topic', required=False, help='Topic to send Log
Messages')
args = parser.parse_args()
- log_file = os.path.join(args.logging_directory, args.logging_file + ".log.0")
+ log_file = os.path.join(args.logging_directory,
+ util.getFullyQualifiedFunctionName(args.tenant,
args.namespace, args.name),
+ "%s-%s.log" % (args.logging_file, args.instance_id))
log.init_rotating_logger(level=logging.INFO, logfile=log_file,
max_files=5, max_bytes=10 * 1024 * 1024)
diff --git a/pulsar-functions/instance/src/main/python/util.py
b/pulsar-functions/instance/src/main/python/util.py
index 643cbac..6aa4948 100644
--- a/pulsar-functions/instance/src/main/python/util.py
+++ b/pulsar-functions/instance/src/main/python/util.py
@@ -64,3 +64,6 @@ def import_class_from_path(from_path, full_class_name):
Log.info("Import failed class_name %s from path %s" % (class_name,
from_path))
Log.info(e, exc_info=True)
return None
+
+def getFullyQualifiedFunctionName(tenant, namespace, name):
+ return "%s/%s/%s" % (tenant, namespace, name)
diff --git
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java
index 6f57ccb..31673d2 100644
---
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java
+++
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java
@@ -79,6 +79,7 @@ public class ProcessRuntimeFactory implements RuntimeFactory {
this.logDirectory =
Paths.get("logs").toFile().getAbsolutePath();
}
}
+ this.logDirectory = this.logDirectory + "/functions";
}
@Override
diff --git
a/pulsar-functions/runtime/src/main/resources/java_instance_log4j2.yml
b/pulsar-functions/runtime/src/main/resources/java_instance_log4j2.yml
index 2f184d1..25583a6 100644
--- a/pulsar-functions/runtime/src/main/resources/java_instance_log4j2.yml
+++ b/pulsar-functions/runtime/src/main/resources/java_instance_log4j2.yml
@@ -116,10 +116,10 @@ Configuration:
-
RollingFile:
name: "Rolling-${ctx:function}"
- fileName :
"${sys:pulsar.log.dir}/functions/${ctx:function}/function.log"
- filePattern :
"${sys:pulsar.log.dir}/functions/${ctx:function}-%d{MM-dd-yyyy}-%i.log.gz"
+ fileName :
"${sys:pulsar.log.dir}/${ctx:function}/${sys:pulsar.log.file}-${ctx:instance}.log"
+ filePattern :
"${sys:pulsar.log.dir}/${ctx:function}/${sys:pulsar.log.file}-${ctx:instance}-%d{MM-dd-yyyy}-%i.log.gz"
PatternLayout:
- Pattern: "%d{ABSOLUTE} %level{length=5} [%thread]
[instance: %X{instance}] %logger{1} - %msg%n"
+ Pattern: "%d{ABSOLUTE} %level{length=5} [%thread]
[instance-%X{instance}] %logger{1} - %msg%n"
Policies:
TimeBasedTriggeringPolicy:
interval: 1
diff --git
a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java
b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java
index e316e6e..d19d57d 100644
---
a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java
+++
b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java
@@ -104,7 +104,7 @@ public class ProcessRuntimeTest {
assertEquals(args.size(), 41);
args.remove(args.size() - 1);
String expectedArgs = "java -cp " + javaInstanceJarFile + "
-Dlog4j.configurationFile=java_instance_log4j2.yml "
- + "-Dpulsar.log.dir=" + logDirectory + " -Dpulsar.log.file=" +
config.getFunctionConfig().getName()
+ + "-Dpulsar.log.dir=" + logDirectory + "/functions" + "
-Dpulsar.log.file=" + config.getFunctionConfig().getName()
+ " org.apache.pulsar.functions.runtime.JavaInstanceMain"
+ " --jar " + userJarFile + " --instance_id "
+ config.getInstanceId() + " --function_id " +
config.getFunctionId()
@@ -133,7 +133,7 @@ public class ProcessRuntimeTest {
args.remove(args.size() - 1);
String expectedArgs = "python " + pythonInstanceFile
+ " --py " + userJarFile + " --logging_directory "
- + logDirectory + " --logging_file " +
config.getFunctionConfig().getName() + " --instance_id "
+ + logDirectory + "/functions" + " --logging_file " +
config.getFunctionConfig().getName() + " --instance_id "
+ config.getInstanceId() + " --function_id " +
config.getFunctionId()
+ " --function_version " + config.getFunctionVersion() + "
--tenant " + config.getFunctionConfig().getTenant()
+ " --namespace " + config.getFunctionConfig().getNamespace()
--
To stop receiving notification emails like this one, please contact
mme...@apache.org.
