This is an automated email from the ASF dual-hosted git repository.
jin pushed a commit to branch fix-slowlog
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git
The following commit(s) were added to refs/heads/fix-slowlog by this push:
new 034cca655 fix(api): refactor/downgrade record logic for slow log
034cca655 is described below
commit 034cca6555ed99e3903b8e618f25b33eeb5a9dd9
Author: imbajin <[email protected]>
AuthorDate: Fri Nov 10 14:42:09 2023 +0800
fix(api): refactor/downgrade record logic for slow log
add some TODOs & assign @SunnyBoy-WYH to address it
---
.../org/apache/hugegraph/api/auth/LoginAPI.java | 56 +++++++--------
.../hugegraph/api/filter/AccessLogFilter.java | 83 +++++++++++-----------
.../apache/hugegraph/api/filter/PathFilter.java | 24 +++----
.../org/apache/hugegraph/metrics/SlowQueryLog.java | 21 ++++--
4 files changed, 92 insertions(+), 92 deletions(-)
diff --git
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/auth/LoginAPI.java
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/auth/LoginAPI.java
index a227fd0b5..0982e01c1 100644
---
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/auth/LoginAPI.java
+++
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/auth/LoginAPI.java
@@ -17,39 +17,40 @@
package org.apache.hugegraph.api.auth;
-import io.swagger.v3.oas.annotations.tags.Tag;
-import jakarta.inject.Singleton;
import javax.security.sasl.AuthenticationException;
-import jakarta.ws.rs.BadRequestException;
-import jakarta.ws.rs.Consumes;
-import jakarta.ws.rs.DELETE;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.HeaderParam;
-import jakarta.ws.rs.NotAuthorizedException;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.core.Context;
-import jakarta.ws.rs.core.HttpHeaders;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hugegraph.core.GraphManager;
-import org.apache.hugegraph.define.Checkable;
-import org.slf4j.Logger;
-
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.api.API;
import org.apache.hugegraph.api.filter.AuthenticationFilter;
import org.apache.hugegraph.api.filter.StatusFilter.Status;
import org.apache.hugegraph.auth.AuthConstant;
import org.apache.hugegraph.auth.UserWithRole;
+import org.apache.hugegraph.core.GraphManager;
+import org.apache.hugegraph.define.Checkable;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.Log;
+import org.slf4j.Logger;
+
import com.codahale.metrics.annotation.Timed;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableMap;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.inject.Singleton;
+import jakarta.ws.rs.BadRequestException;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.HeaderParam;
+import jakarta.ws.rs.NotAuthorizedException;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.HttpHeaders;
+
@Path("graphs/{graph}/auth")
@Singleton
@Tag(name = "LoginAPI")
@@ -63,8 +64,7 @@ public class LoginAPI extends API {
@Status(Status.OK)
@Consumes(APPLICATION_JSON)
@Produces(APPLICATION_JSON_WITH_CHARSET)
- public String login(@Context GraphManager manager,
- @PathParam("graph") String graph,
+ public String login(@Context GraphManager manager, @PathParam("graph")
String graph,
JsonLogin jsonLogin) {
LOG.debug("Graph [{}] user login: {}", graph, jsonLogin);
checkCreatingBody(jsonLogin);
@@ -94,13 +94,10 @@ public class LoginAPI extends API {
LOG.debug("Graph [{}] user logout: {}", graph, auth);
if (!auth.startsWith(AuthenticationFilter.BEARER_TOKEN_PREFIX)) {
- throw new BadRequestException(
- "Only HTTP Bearer authentication is supported");
+ throw new BadRequestException("Only HTTP Bearer authentication is
supported");
}
- String token = auth.substring(AuthenticationFilter.BEARER_TOKEN_PREFIX
- .length());
-
+ String token =
auth.substring(AuthenticationFilter.BEARER_TOKEN_PREFIX.length());
manager.authManager().logoutUser(token);
}
@@ -119,12 +116,10 @@ public class LoginAPI extends API {
LOG.debug("Graph [{}] get user: {}", graph, token);
if (!token.startsWith(AuthenticationFilter.BEARER_TOKEN_PREFIX)) {
- throw new BadRequestException(
- "Only HTTP Bearer authentication is supported");
+ throw new BadRequestException("Only HTTP Bearer authentication is
supported");
}
- token = token.substring(AuthenticationFilter.BEARER_TOKEN_PREFIX
- .length());
+ token =
token.substring(AuthenticationFilter.BEARER_TOKEN_PREFIX.length());
UserWithRole userWithRole = manager.authManager().validateUser(token);
HugeGraph g = graph(manager, graph);
@@ -144,8 +139,7 @@ public class LoginAPI extends API {
@Override
public void checkCreate(boolean isBatch) {
- E.checkArgument(!StringUtils.isEmpty(this.name),
- "The name of user can't be null");
+ E.checkArgument(!StringUtils.isEmpty(this.name), "The name of user
can't be null");
E.checkArgument(!StringUtils.isEmpty(this.password),
"The password of user can't be null");
}
diff --git
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java
index 3b529cf0a..e57a88ce5 100644
---
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java
+++
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java
@@ -17,7 +17,6 @@
package org.apache.hugegraph.api.filter;
-import static org.apache.hugegraph.api.filter.PathFilter.REQUEST_PARAMS_JSON;
import static org.apache.hugegraph.api.filter.PathFilter.REQUEST_TIME;
import static
org.apache.hugegraph.metrics.MetricsUtil.METRICS_PATH_FAILED_COUNTER;
import static
org.apache.hugegraph.metrics.MetricsUtil.METRICS_PATH_RESPONSE_TIME_HISTOGRAM;
@@ -25,12 +24,11 @@ import static
org.apache.hugegraph.metrics.MetricsUtil.METRICS_PATH_SUCCESS_COUN
import static
org.apache.hugegraph.metrics.MetricsUtil.METRICS_PATH_TOTAL_COUNTER;
import java.io.IOException;
+import java.net.URI;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.config.ServerOptions;
import org.apache.hugegraph.metrics.MetricsUtil;
-import org.apache.hugegraph.metrics.SlowQueryLog;
-import org.apache.hugegraph.util.JsonUtil;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
@@ -42,12 +40,12 @@ import jakarta.ws.rs.container.ContainerResponseFilter;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.ext.Provider;
-
+// TODO: should add test for this class
@Provider
@Singleton
public class AccessLogFilter implements ContainerResponseFilter {
- private static final String DELIMETER = "/";
+ private static final String DELIMITER = "/";
private static final String GRAPHS = "graphs";
private static final String GREMLIN = "gremlin";
private static final String CYPHER = "cypher";
@@ -57,6 +55,25 @@ public class AccessLogFilter implements
ContainerResponseFilter {
@Context
private jakarta.inject.Provider<HugeConfig> configProvider;
+ public static boolean needRecord(ContainerRequestContext context) {
+ // TODO: add test for 'path' result ('/gremlin' or 'gremlin')
+ String path = context.getUriInfo().getPath();
+
+ // GraphsAPI/CypherAPI/Job GremlinAPI
+ if (path.startsWith(GRAPHS)) {
+ if (HttpMethod.GET.equals(context.getMethod()) ||
+ path.endsWith(CYPHER) || path.endsWith(GREMLIN)) {
+ return true;
+ }
+ }
+ // Raw GremlinAPI
+ return path.startsWith(GREMLIN);
+ }
+
+ private String join(String path1, String path2) {
+ return String.join(DELIMITER, path1, path2);
+ }
+
/**
* Use filter to log request info
*
@@ -64,10 +81,12 @@ public class AccessLogFilter implements
ContainerResponseFilter {
* @param responseContext responseContext
*/
@Override
- public void filter(ContainerRequestContext requestContext,
ContainerResponseContext responseContext) throws IOException {
+ public void filter(ContainerRequestContext requestContext,
+ ContainerResponseContext responseContext) throws
IOException {
// Grab corresponding request / response info from context;
- String method = requestContext.getRequest().getMethod();
- String path = requestContext.getUriInfo().getPath();
+ URI uri = requestContext.getUriInfo().getRequestUri();
+ String path = uri.getRawPath();
+ String method = requestContext.getMethod();
String metricsName = join(path, method);
MetricsUtil.registerCounter(join(metricsName,
METRICS_PATH_TOTAL_COUNTER)).inc();
@@ -77,48 +96,30 @@ public class AccessLogFilter implements
ContainerResponseFilter {
MetricsUtil.registerCounter(join(metricsName,
METRICS_PATH_FAILED_COUNTER)).inc();
}
- // get responseTime
Object requestTime = requestContext.getProperty(REQUEST_TIME);
- if(requestTime != null){
+ if (requestTime != null) {
long now = System.currentTimeMillis();
long start = (Long) requestTime;
- long responseTime = now - start;
+ long executeTime = now - start;
- MetricsUtil.registerHistogram(
- join(metricsName,
METRICS_PATH_RESPONSE_TIME_HISTOGRAM))
- .update(responseTime);
+ MetricsUtil.registerHistogram(join(metricsName,
METRICS_PATH_RESPONSE_TIME_HISTOGRAM))
+ .update(executeTime);
- HugeConfig config = configProvider.get();
- long timeThreshold =
config.get(ServerOptions.SLOW_QUERY_LOG_TIME_THRESHOLD);
+ if (needRecord(requestContext)) {
+ HugeConfig config = configProvider.get();
+ long timeThreshold =
config.get(ServerOptions.SLOW_QUERY_LOG_TIME_THRESHOLD);
- // record slow query log
- if (timeThreshold > 0 && isSlowQueryLogWhiteAPI(requestContext) &&
responseTime > timeThreshold) {
- SlowQueryLog log = new SlowQueryLog(responseTime, start,
(String) requestContext.getProperty(REQUEST_PARAMS_JSON),
- method, timeThreshold,
path);
- LOG.info("Slow query: {}", JsonUtil.toJson(log));
+ // Record slow query if meet needs
+ if (timeThreshold > 0 && executeTime > timeThreshold) {
+ // TODO: set RequsetBody null, handle it later & should
record "client IP"
+ LOG.info("[Slow Query] execTime={}ms, body={}, method={},
path={}, query={}",
+ executeTime, null, method, path,
uri.getRawQuery());
+ }
}
}
}
- private String join(String path1, String path2) {
- return String.join(DELIMETER, path1, path2);
- }
-
- private boolean statusOk(int status){
- return status == 200 || status == 201 || status == 202;
- }
-
- public static boolean isSlowQueryLogWhiteAPI(ContainerRequestContext
context) {
- String path = context.getUriInfo().getPath();
- String method = context.getRequest().getMethod();
-
- // GraphsAPI/CypherAPI/Job GremlinAPI
- if (path.startsWith(GRAPHS)) {
- if (method.equals(HttpMethod.GET) || path.endsWith(CYPHER) ||
path.endsWith(GREMLIN) ){
- return true;
- }
- }
- // Raw GremlinAPI
- return path.startsWith(GREMLIN);
+ private boolean statusOk(int status) {
+ return status >= 200 && status < 300;
}
}
diff --git
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java
index e1e449ef2..3ae9c35bc 100644
---
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java
+++
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java
@@ -17,20 +17,12 @@
package org.apache.hugegraph.api.filter;
-import static org.apache.hugegraph.api.API.CHARSET;
-
import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.io.Charsets;
-import org.apache.commons.io.IOUtils;
import jakarta.inject.Singleton;
-import jakarta.ws.rs.HttpMethod;
import jakarta.ws.rs.container.ContainerRequestContext;
import jakarta.ws.rs.container.ContainerRequestFilter;
import jakarta.ws.rs.container.PreMatching;
-import jakarta.ws.rs.core.MultivaluedMap;
import jakarta.ws.rs.ext.Provider;
@Provider
@@ -42,23 +34,25 @@ public class PathFilter implements ContainerRequestFilter {
public static final String REQUEST_PARAMS_JSON = "request_params_json";
@Override
- public void filter(ContainerRequestContext context)
- throws IOException {
+ public void filter(ContainerRequestContext context) throws IOException {
context.setProperty(REQUEST_TIME, System.currentTimeMillis());
- // record the request json
+ // TODO: comment it to fix loader bug, handle it later
+ /*// record the request json
String method = context.getMethod();
String requestParamsJson = "";
if (method.equals(HttpMethod.POST)) {
- requestParamsJson = IOUtils.toString(context.getEntityStream(),
Charsets.toCharset(CHARSET));
+ requestParamsJson = IOUtils.toString(context.getEntityStream(),
+ Charsets.toCharset(CHARSET));
// replace input stream because we have already read it
InputStream in = IOUtils.toInputStream(requestParamsJson,
Charsets.toCharset(CHARSET));
context.setEntityStream(in);
- } else if(method.equals(HttpMethod.GET)){
- MultivaluedMap<String, String> pathParameters =
context.getUriInfo().getPathParameters();
+ } else if (method.equals(HttpMethod.GET)) {
+ MultivaluedMap<String, String> pathParameters =
context.getUriInfo()
+
.getPathParameters();
requestParamsJson = pathParameters.toString();
}
- context.setProperty(REQUEST_PARAMS_JSON, requestParamsJson);
+ context.setProperty(REQUEST_PARAMS_JSON, requestParamsJson);*/
}
}
diff --git
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/metrics/SlowQueryLog.java
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/metrics/SlowQueryLog.java
index cb3f1c712..af49e5fdf 100644
---
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/metrics/SlowQueryLog.java
+++
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/metrics/SlowQueryLog.java
@@ -19,20 +19,20 @@ package org.apache.hugegraph.metrics;
public class SlowQueryLog {
- public Long executeTime;
+ public long executeTime;
- public Long startTime;
+ public long startTime;
public String rawQuery;
public String method;
- public Long threshold;
+ public long threshold;
public String path;
- public SlowQueryLog(Long executeTime, Long startTime, String rawQuery,
String method, Long threshold,
- String path) {
+ public SlowQueryLog(long executeTime, long startTime, String rawQuery,
String method,
+ long threshold, String path) {
this.executeTime = executeTime;
this.startTime = startTime;
this.rawQuery = rawQuery;
@@ -40,4 +40,15 @@ public class SlowQueryLog {
this.threshold = threshold;
this.path = path;
}
+
+ @Override
+ public String toString() {
+ return "SlowQueryLog{executeTime=" + executeTime +
+ ", startTime=" + startTime +
+ ", rawQuery='" + rawQuery + '\'' +
+ ", method='" + method + '\'' +
+ ", threshold=" + threshold +
+ ", path='" + path + '\'' +
+ '}';
+ }
}
