This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new ad8f468 IMPALA-8860: Improve /log_level usability on WebUI
ad8f468 is described below
commit ad8f468871d3a893bf4c7b702025ec70765ce8e1
Author: Zoltan Garaguly <zgarag...@cloudera.com>
AuthorDate: Tue May 12 15:20:08 2020 +0200
IMPALA-8860: Improve /log_level usability on WebUI
Add glog level fetching logic and fetch glog level on every request
which allows:
- showing glog level on page load/reload
- showing current glog level in "Log level" combo box
- showing current glog level in text format
Add log4j log levels fetching logic and fetch all java class log levels
on every request:
- log4j levels for all java classes previously set are shown on page
as a list, fetching of individual class log levels not needed
anymore
Page layout standardization:
- glog/log4j part has similar layout
- using terms of frontend/backend logs
Change-Id: I2fbf2ef21f4af297913a4e9b16a391768624da33
Reviewed-on: http://gerrit.cloudera.org:8080/15903
Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
---
be/src/util/logging-support.cc | 80 +++++++++++-----------
common/thrift/Logging.thrift | 18 ++---
.../java/org/apache/impala/util/GlogAppender.java | 36 ++++++----
tests/webserver/test_web_pages.py | 41 ++++-------
www/log_level.tmpl | 53 +++++++-------
5 files changed, 113 insertions(+), 115 deletions(-)
diff --git a/be/src/util/logging-support.cc b/be/src/util/logging-support.cc
index 9d0232d..d05d8dc 100644
--- a/be/src/util/logging-support.cc
+++ b/be/src/util/logging-support.cc
@@ -86,7 +86,7 @@ int FLAGS_v_original_value;
static jclass log4j_logger_class_;
// Jni method descriptors corresponding to getLogLevel() and setLogLevel()
operations.
-static jmethodID get_log_level_method; // GlogAppender.getLogLevel()
+static jmethodID get_log_levels_method; // GlogAppender.getLogLevels()
static jmethodID set_log_level_method; // GlogAppender.setLogLevel()
static jmethodID reset_log_levels_method; // GlogAppender.resetLogLevels()
@@ -98,27 +98,18 @@ void AddDocumentMember(const string& message, const char*
member,
document->AddMember(key, output, document->GetAllocator());
}
-template<class F>
-Webserver::UrlCallback MakeCallback(const F& fnc, bool display_log4j_handlers)
{
- return [fnc, display_log4j_handlers](const auto& req, auto* doc) {
- // Display log4j log level handlers only when display_log4j_handlers is
true.
- if (display_log4j_handlers) AddDocumentMember("true",
"include_log4j_handlers", doc);
- (*fnc)(req, doc);
- };
-}
-
void InitDynamicLoggingSupport() {
JNIEnv* env = JniUtil::GetJNIEnv();
ABORT_IF_ERROR(JniUtil::GetGlobalClassRef(env,
"org/apache/impala/util/GlogAppender",
&log4j_logger_class_));
- JniMethodDescriptor get_log_level_method_desc =
- {"getLogLevel", "([B)Ljava/lang/String;", &get_log_level_method};
+ JniMethodDescriptor get_log_levels_method_desc =
+ {"getLogLevels", "()[B", &get_log_levels_method};
JniMethodDescriptor set_log_level_method_desc =
{"setLogLevel", "([B)Ljava/lang/String;", &set_log_level_method};
JniMethodDescriptor reset_log_level_method_desc =
{"resetLogLevels", "()V", &reset_log_levels_method};
ABORT_IF_ERROR(JniUtil::LoadStaticJniMethod(
- env, log4j_logger_class_, &get_log_level_method_desc));
+ env, log4j_logger_class_, &get_log_levels_method_desc));
ABORT_IF_ERROR(JniUtil::LoadStaticJniMethod(
env, log4j_logger_class_, &set_log_level_method_desc));
ABORT_IF_ERROR(JniUtil::LoadStaticJniMethod(
@@ -132,38 +123,28 @@ void InitDynamicLoggingSupport() {
[](const char* flagname, int value) { return value >= 0 && value <= 3;
});
}
-// Helper method to get the log level of given Java class. It is a JNI wrapper
around
-// GlogAppender.getLogLevel().
-Status GetJavaLogLevel(const TGetJavaLogLevelParams& params, string* result) {
- return JniCall::static_method(log4j_logger_class_, get_log_level_method)
- .with_thrift_arg(params).Call(result);
-}
-
Status ResetJavaLogLevels() {
return JniCall::static_method(log4j_logger_class_,
reset_log_levels_method).Call();
}
-// Callback handler for /get_java_loglevel.
-void GetJavaLogLevelCallback(const Webserver::WebRequest& req, Document*
document) {
- const auto& args = req.parsed_args;
- Webserver::ArgumentMap::const_iterator log_getclass = args.find("class");
- if (log_getclass == args.end() || log_getclass->second.empty()) {
- AddDocumentMember("Invalid input class name", "error", document);
- return;
- }
- string result;
- TGetJavaLogLevelParams params;
- params.__set_class_name(log_getclass->second);
- Status status = GetJavaLogLevel(params, &result);
+void GetJavaLogLevels(Document* document) {
+ TGetJavaLogLevelsResult result;
+
+ Status status =
+ JniCall::static_method(log4j_logger_class_,
get_log_levels_method).Call(&result);
+
if (!status.ok()) {
AddDocumentMember(status.GetDetail(), "error", document);
return;
}
- if (result.empty()) {
- AddDocumentMember("Invalid input class name", "error", document);
- return;
+
+ string log_levels;
+ for (const string& logLevel: result.log_levels) {
+ log_levels.append(logLevel);
+ log_levels.append("\n");
}
- AddDocumentMember(result, "get_java_loglevel_result", document);
+
+ AddDocumentMember(log_levels, "get_java_loglevel_result", document);
}
// Callback handler for /set_java_loglevel.
@@ -205,6 +186,19 @@ void ResetJavaLogLevelCallback(const
Webserver::WebRequest& req, Document* docum
AddDocumentMember("Java log levels reset.", "reset_java_loglevel_result",
document);
}
+void GetGlogLevel(Document* document) {
+ string log_level;
+
+ if (google::GetCommandLineOption("v", &log_level)) {
+ AddDocumentMember(log_level, "glog_level", document);
+ }
+ else {
+ AddDocumentMember(to_string(FLAGS_v_original_value), "glog_level",
document);
+ }
+
+ AddDocumentMember(to_string(FLAGS_v_original_value), "default_glog_level",
document);
+}
+
// Callback handler for /set_glog_level
void SetGlogLevelCallback(const Webserver::WebRequest& req, Document*
document) {
const auto& args = req.parsed_args;
@@ -221,7 +215,6 @@ void SetGlogLevelCallback(const Webserver::WebRequest& req,
Document* document)
return;
}
AddDocumentMember(new_log_level, "set_glog_level_result", document);
- AddDocumentMember(to_string(FLAGS_v_original_value), "default_glog_level",
document);
}
// Callback handler for /reset_glog_level
@@ -231,8 +224,19 @@ void ResetGlogLevelCallback(const Webserver::WebRequest&
req, Document* document
AddDocumentMember(new_log_level, "reset_glog_level_result", document);
}
+template<class F>
+Webserver::UrlCallback MakeCallback(const F& fnc, bool display_log4j_handlers)
{
+ return [fnc, display_log4j_handlers](const auto& req, auto* doc) {
+ // Display log4j log level handlers only when display_log4j_handlers is
true.
+ if (display_log4j_handlers) AddDocumentMember("true",
"include_log4j_handlers", doc);
+ (*fnc)(req, doc);
+ if (display_log4j_handlers) {GetJavaLogLevels(doc);}
+ GetGlogLevel(doc);
+ };
}
+} // namespace
+
namespace impala {
void InitJvmLoggingSupport() {
@@ -321,8 +325,6 @@ void RegisterLogLevelCallbacks(Webserver* webserver, bool
register_log4j_handler
webserver->RegisterUrlCallback("/reset_glog_level", "log_level.tmpl",
MakeCallback(&ResetGlogLevelCallback, register_log4j_handlers), false);
if (!register_log4j_handlers) return;
- webserver->RegisterUrlCallback("/get_java_loglevel", "log_level.tmpl",
- MakeCallback(&GetJavaLogLevelCallback, register_log4j_handlers), false);
webserver->RegisterUrlCallback("/set_java_loglevel", "log_level.tmpl",
MakeCallback(&SetJavaLogLevelCallback, register_log4j_handlers), false);
webserver->RegisterUrlCallback("/reset_java_loglevel", "log_level.tmpl",
diff --git a/common/thrift/Logging.thrift b/common/thrift/Logging.thrift
index b7c51ae..6335667 100644
--- a/common/thrift/Logging.thrift
+++ b/common/thrift/Logging.thrift
@@ -31,15 +31,17 @@ enum TLogLevel {
FATAL = 6
}
-// Helper structs for GetJavaLogLevel(), SetJavaLogLevel() methods.
-// These are used as input params to get/set the logging level of a
-// particular Java class at runtime using GlogAppender.getLogLevel()
-// and GlogAppender.setLogLevel() methods.
-struct TGetJavaLogLevelParams {
- 1: required string class_name
-}
-
+// Helper structs for GetJavaLogLevels(), SetJavaLogLevel() methods.
+// These are used as:
+// - input param to set the logging level of a particular Java class
+// at runtime using GlogAppender.setLogLevel()
+// - return value as a list of Java classes with corresponding
+// logging levels set before using GlogAppender.getLogLevels()
struct TSetJavaLogLevelParams {
1: required string class_name
2: required string log_level
}
+
+struct TGetJavaLogLevelsResult {
+ 1: required list<string> log_levels
+}
diff --git a/fe/src/main/java/org/apache/impala/util/GlogAppender.java
b/fe/src/main/java/org/apache/impala/util/GlogAppender.java
index 2eb33f3..82c6c79 100644
--- a/fe/src/main/java/org/apache/impala/util/GlogAppender.java
+++ b/fe/src/main/java/org/apache/impala/util/GlogAppender.java
@@ -17,6 +17,9 @@
package org.apache.impala.util;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
import java.util.Properties;
import org.apache.log4j.AppenderSkeleton;
@@ -30,7 +33,7 @@ import org.apache.impala.common.InternalException;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.JniUtil;
import org.apache.impala.service.BackendConfig;
-import org.apache.impala.thrift.TGetJavaLogLevelParams;
+import org.apache.impala.thrift.TGetJavaLogLevelsResult;
import org.apache.impala.thrift.TSetJavaLogLevelParams;
import org.apache.impala.thrift.TLogLevel;
import org.apache.thrift.protocol.TBinaryProtocol;
@@ -139,17 +142,6 @@ public class GlogAppender extends AppenderSkeleton {
}
/**
- * Get the log4j log level corresponding to a serialized
TGetJavaLogLevelParams.
- */
- public static String getLogLevel(byte[] serializedParams) throws
ImpalaException {
- TGetJavaLogLevelParams thriftParams = new TGetJavaLogLevelParams();
- JniUtil.deserializeThrift(protocolFactory_, thriftParams,
serializedParams);
- String className = thriftParams.getClass_name();
- if (Strings.isNullOrEmpty(className)) return null;
- return Logger.getLogger(className).getEffectiveLevel().toString();
- }
-
- /**
* Sets the logging level of a class as per serialized
TSetJavaLogLevelParams.
*/
public static String setLogLevel(byte[] serializedParams) throws
ImpalaException {
@@ -171,4 +163,24 @@ public class GlogAppender extends AppenderSkeleton {
Install(TLogLevel.values()[BackendConfig.INSTANCE.getImpalaLogLevel()],
TLogLevel.values()[BackendConfig.INSTANCE.getNonImpalaJavaVlogLevel()]);
}
+
+ /**
+ * Get all the previously set log4j log levels.
+ */
+ public static byte[] getLogLevels() throws ImpalaException {
+ Enumeration<Logger> allLoggers = LogManager.getCurrentLoggers();
+ List<String> logLevels = new ArrayList<String>();
+
+ while (allLoggers.hasMoreElements()) {
+ Logger logger = allLoggers.nextElement();
+ if (logger.getLevel() != null) {
+ logLevels.add(logger.getName() + " : " + logger.getLevel());
+ }
+ }
+
+ TGetJavaLogLevelsResult result = new TGetJavaLogLevelsResult();
+ result.setLog_levels(logLevels);
+
+ return JniUtil.serializeToThrift(result, protocolFactory_);
+ }
};
diff --git a/tests/webserver/test_web_pages.py
b/tests/webserver/test_web_pages.py
index 6fa6cea..b93dfa1 100644
--- a/tests/webserver/test_web_pages.py
+++ b/tests/webserver/test_web_pages.py
@@ -31,7 +31,6 @@ import requests
class TestWebPage(ImpalaTestSuite):
ROOT_URL = "http://localhost:{0}/";
- GET_JAVA_LOGLEVEL_URL = "http://localhost:{0}/get_java_loglevel";
SET_JAVA_LOGLEVEL_URL = "http://localhost:{0}/set_java_loglevel";
RESET_JAVA_LOGLEVEL_URL = "http://localhost:{0}/reset_java_loglevel";
SET_GLOG_LOGLEVEL_URL = "http://localhost:{0}/set_glog_level";
@@ -201,49 +200,31 @@ class TestWebPage(ImpalaTestSuite):
malformed inputs. This however does not test that the log level changes
are actually
in effect."""
# Check that the log_level end points are accessible.
- self.get_and_check_status_jvm(self.GET_JAVA_LOGLEVEL_URL)
self.get_and_check_status_jvm(self.SET_JAVA_LOGLEVEL_URL)
self.get_and_check_status_jvm(self.RESET_JAVA_LOGLEVEL_URL)
self.get_and_check_status(self.SET_GLOG_LOGLEVEL_URL)
self.get_and_check_status(self.RESET_GLOG_LOGLEVEL_URL)
- # Try getting log level of a class.
- get_loglevel_url = (self.GET_JAVA_LOGLEVEL_URL + "?class" +
- "=org.apache.impala.catalog.HdfsTable")
- self.get_and_check_status_jvm(get_loglevel_url, "DEBUG")
# Set the log level of a class to TRACE and confirm the setting is in place
set_loglevel_url = (self.SET_JAVA_LOGLEVEL_URL + "?class" +
"=org.apache.impala.catalog.HdfsTable&level=trace")
- self.get_and_check_status_jvm(set_loglevel_url, "Effective log level:
TRACE")
+ self.get_and_check_status_jvm(
+ set_loglevel_url,
+ "org.apache.impala.catalog.HdfsTable : TRACE")
- get_loglevel_url = (self.GET_JAVA_LOGLEVEL_URL + "?class" +
- "=org.apache.impala.catalog.HdfsTable")
- self.get_and_check_status_jvm(get_loglevel_url, "TRACE")
- # Check the log level of a different class and confirm it is still DEBUG
- get_loglevel_url = (self.GET_JAVA_LOGLEVEL_URL + "?class" +
- "=org.apache.impala.catalog.HdfsPartition")
- self.get_and_check_status_jvm(get_loglevel_url, "DEBUG")
-
- # Reset Java logging levels and check the logging level of the class again
+ # Reset Java logging levels
self.get_and_check_status_jvm(self.RESET_JAVA_LOGLEVEL_URL, "Java log
levels reset.")
- get_loglevel_url = (self.GET_JAVA_LOGLEVEL_URL + "?class" +
- "=org.apache.impala.catalog.HdfsTable")
- self.get_and_check_status_jvm(get_loglevel_url, "DEBUG")
# Set a new glog level and make sure the setting has been applied.
set_glog_url = (self.SET_GLOG_LOGLEVEL_URL + "?glog=3")
- self.get_and_check_status(set_glog_url, "v set to 3")
+ self.get_and_check_status(set_glog_url, "val(3)")
# Try resetting the glog logging defaults again.
- self.get_and_check_status(self.RESET_GLOG_LOGLEVEL_URL, "v set to ")
-
- # Try to get the log level of an empty class input
- get_loglevel_url = (self.GET_JAVA_LOGLEVEL_URL + "?class=")
- self.get_and_check_status_jvm(get_loglevel_url)
+ self.get_and_check_status(self.RESET_GLOG_LOGLEVEL_URL, "Current backend
log level: ")
# Same as above, for set log level request
set_loglevel_url = (self.SET_JAVA_LOGLEVEL_URL + "?class=")
- self.get_and_check_status_jvm(get_loglevel_url)
+ self.get_and_check_status_jvm(set_loglevel_url)
# Empty input for setting a glog level request
set_glog_url = (self.SET_GLOG_LOGLEVEL_URL + "?glog=")
@@ -254,7 +235,9 @@ class TestWebPage(ImpalaTestSuite):
# Level.toLevel() method.
set_loglevel_url = (self.SET_JAVA_LOGLEVEL_URL + "?class" +
"=org.apache.impala.catalog.HdfsTable&level=foo&")
- self.get_and_check_status_jvm(set_loglevel_url, "Effective log level:
DEBUG")
+ self.get_and_check_status_jvm(
+ set_loglevel_url,
+ "org.apache.impala.catalog.HdfsTable : DEBUG")
# Try setting an invalid glog level.
set_glog_url = self.SET_GLOG_LOGLEVEL_URL + "?glog=foo"
@@ -274,11 +257,11 @@ class TestWebPage(ImpalaTestSuite):
self.get_and_check_status(self.RESET_GLOG_LOGLEVEL_URL)
# Set log level to 3.
set_glog_url = (self.SET_GLOG_LOGLEVEL_URL + "?glog=3")
- self.get_and_check_status(set_glog_url, "v set to 3")
+ self.get_and_check_status(set_glog_url, "val(3)")
# Check that Impala doesn't crash when running a query that aggregates.
self.client.execute("select avg(int_col) from functional.alltypessmall")
# Reset log level.
- self.get_and_check_status(self.RESET_GLOG_LOGLEVEL_URL, "v set to ")
+ self.get_and_check_status(self.RESET_GLOG_LOGLEVEL_URL, "Current backend
log level: ")
def test_catalog(self, cluster_properties, unique_database):
"""Tests the /catalog and /catalog_object endpoints."""
diff --git a/www/log_level.tmpl b/www/log_level.tmpl
index f3b3059..7cd9772 100644
--- a/www/log_level.tmpl
+++ b/www/log_level.tmpl
@@ -26,15 +26,10 @@ under the License.
</style>
{{?include_log4j_handlers}}
-<h2>Change Java log level (log4j)</h2>
+<h2>Frontend log level configuration (log4j)</h2>
<div class="log-level">
-<form action="get_java_loglevel">{{>www/form-hidden-inputs.tmpl}}
- <div class="form-group">
- <input type="text" class="form-control" name="class" placeholder="e.g.
org.apache.impala.analysis.Analyzer">
- </div>
- <button type="submit" class="btn btn-primary btn-sm">Get Java Log
Level</button>
- <label>{{get_java_loglevel_result}}</label>
-</form>
+<h5>Current frontend log level:</h5>
+<span style="white-space: pre-line">{{get_java_loglevel_result}}</span>
<br>
<form action="set_java_loglevel">{{>www/form-hidden-inputs.tmpl}}
<div class="form-group" name="level">
@@ -52,46 +47,50 @@ under the License.
<option value="trace">TRACE</option>
<option value="warn">WARN</option>
</select>
- <button type="submit" class="btn btn-primary btn-sm">Set Java Log
Level</button>
- {{?set_java_loglevel_result}}
- <strong> Effective log level: {{set_java_loglevel_result}}</strong>
- {{/set_java_loglevel_result}}
+ <button type="submit" class="btn btn-primary btn-sm">Set Frontend Log
Level</button>
</div>
</div>
</form>
<form action="reset_java_loglevel">{{>www/form-hidden-inputs.tmpl}}
<div class="col-xs-20">
- <button type="submit" class="btn btn-primary btn-sm">Reset Java Log
Levels</button>
+ <button type="submit" class="btn btn-primary btn-sm">Reset Frontend Log
Levels</button>
<strong>{{reset_java_loglevel_result}}</strong>
</div>
</form>
+<br>
{{/include_log4j_handlers}}
-<h2>Change backend logging level (glog)</h2>
+<h2>Backend log level configuration (glog)</h2>
+<h5>Current backend log level: <span id="glog_text"></span></h5>
<form action="set_glog_level">{{>www/form-hidden-inputs.tmpl}}
<div class="form-group" name="level">
<div class="col-xs-20">
<label>Log level:</label>
<select name="glog" class="selectpicker" data-style="btn-primary btn-sm">
- <option value="0">0</option>
- <option value="1">1</option>
- <option value="2">2</option>
- <option value="3">3</option>
+ <option value="0">0 - Off</option>
+ <option value="1">1 - Info</option>
+ <option value="2">2 - Debug</option>
+ <option value="3">3 - All</option>
</select>
- <button type="submit" class="btn btn-primary btn-sm">Set Glog
Level</button>
- {{?set_glog_level_result}}
- <strong>    Glog level for --{{set_glog_level_result}}</strong>
- <strong>    Default glog level: {{default_glog_level}}</strong>
- {{/set_glog_level_result}}
+ <button type="submit" class="btn btn-primary btn-sm">Set Backend Log
Level</button>
</div>
</div>
</form>
<form action="reset_glog_level">{{>www/form-hidden-inputs.tmpl}}
<div class="col-xs-20">
- <button type="submit" class="btn btn-primary btn-sm">Reset Glog
Levels</button>
- {{?reset_glog_level_result}}
- <strong>     Glog logging level for
--{{reset_glog_level_result}}</strong>
- {{/reset_glog_level_result}}
+ <button type="submit" class="btn btn-primary btn-sm">Reset Backend Log
Level</button>
+ <strong>    Default backend log level:
{{default_glog_level}}</strong>
</div>
</form>
+
+<script>
+
+ $(document).ready(function() {
+ $('select[name=glog]').val({{glog_level}});
+ var text=$('select[name=glog]').children("option:selected").text();
+ document.getElementById("glog_text").innerHTML = text;
+ });
+
+</script>
+
{{>www/common-footer.tmpl}}
