kezhenxu94 commented on code in PR #10884: URL: https://github.com/apache/skywalking/pull/10884#discussion_r1223916494
########## oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/uri/quickmatch/PatternTree.java: ########## @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.uri.quickmatch; + +import java.util.ArrayList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.apache.skywalking.oap.server.library.util.StringFormatGroup; + +@EqualsAndHashCode +@ToString +public class PatternTree { + private final List<PatternToken> roots; + + public PatternTree() { + roots = new ArrayList<>(); + } + + /** + * The pattern is split by /, and each token is a node in the tree. Each node either a literal string or "{var}" + * representing a variable. + * + * @param pattern of URIs + */ + public void addPattern(String pattern) { + final String[] tokens = pattern.split("/"); + + PatternToken current = null; + for (final PatternToken patternToken : roots) { + if (patternToken.isMatch(tokens[0])) { + current = patternToken; + break; + } + } + + if (current == null) { + current = new StringToken(tokens[0]); + roots.add(current); + } + + if (tokens.length == 1) { + current.setExpression(pattern); + return; + } + + for (int i = 1; i < tokens.length; i++) { + final String token = tokens[i]; + PatternToken newToken; + if (VarToken.VAR_TOKEN.equals(token)) { + newToken = new VarToken(); Review Comment: Correct me if I am wrong, looks like we can have a singleton for `VarToken` and don't have to recreate `VarToken` every time. ########## oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/uri/quickmatch/PatternTree.java: ########## @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.uri.quickmatch; + +import java.util.ArrayList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.apache.skywalking.oap.server.library.util.StringFormatGroup; + +@EqualsAndHashCode +@ToString +public class PatternTree { + private final List<PatternToken> roots; + + public PatternTree() { + roots = new ArrayList<>(); + } + + /** + * The pattern is split by /, and each token is a node in the tree. Each node either a literal string or "{var}" + * representing a variable. + * + * @param pattern of URIs + */ + public void addPattern(String pattern) { + final String[] tokens = pattern.split("/"); + + PatternToken current = null; + for (final PatternToken patternToken : roots) { + if (patternToken.isMatch(tokens[0])) { + current = patternToken; + break; + } + } + + if (current == null) { + current = new StringToken(tokens[0]); + roots.add(current); + } + + if (tokens.length == 1) { + current.setExpression(pattern); + return; + } + + for (int i = 1; i < tokens.length; i++) { + final String token = tokens[i]; + PatternToken newToken; + if (VarToken.VAR_TOKEN.equals(token)) { + newToken = new VarToken(); + } else { + newToken = new StringToken(token); + } + final PatternToken found = current.find(newToken); + if (found == null) { + current = current.add(newToken); + } else { + current = found; + } + } + current.setExpression(pattern); + } + + public StringFormatGroup.FormatResult match(String uri) { + final String[] slices = uri.split("/"); + List<PatternToken> current = roots; + PatternToken matchedToken = null; + for (final String slice : slices) { + boolean matched = false; + for (final PatternToken patternToken : current) { + if (patternToken.isMatch(slice)) { + matchedToken = patternToken; + matched = true; + break; + } + } + if (!matched) { + return new StringFormatGroup.FormatResult(false, uri, null); + } + current = matchedToken.children(); + } + if (matchedToken.isLeaf()) { Review Comment: `matchedToken` might be null ```suggestion if (matchedToken != null && matchedToken.isLeaf()) { ``` ########## oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java: ########## @@ -69,6 +128,77 @@ private String formatByOpenapi(String serviceName, String endpointName) { log.trace("Endpoint {} of Service {} keeps unchanged.", endpointName, serviceName); } } - return formatResult.getName(); + return new Tuple2<>(formatResult.getName(), formatResult.isMatch()); + } + + private Tuple2<String, Boolean> formatByQuickUriPattern(String serviceName, String endpointName) { + final StringFormatGroup.FormatResult formatResult = quickUriGroupingRule.format(serviceName, endpointName); + if (log.isDebugEnabled() || log.isTraceEnabled()) { + if (formatResult.isMatch()) { + log.debug( + "Endpoint {} of Service {} has been renamed in group {} by AI/ML-powered URI pattern recognition", + endpointName, serviceName, formatResult.getName() + ); + } else { + log.trace("Endpoint {} of Service {} keeps unchanged.", endpointName, serviceName); + } + } + return new Tuple2<>(formatResult.getName(), formatResult.isMatch()); + } + + public void startHttpUriRecognitionSvr(final HttpUriRecognition httpUriRecognitionSvr, + final MetadataQueryService metadataQueryService, + int maxEndpointCandidatePerSvr) { + this.maxHttpUrisNumberPerService = maxEndpointCandidatePerSvr; + if (!httpUriRecognitionSvr.isInitialized()) { + return; + } + this.quickUriGroupingRule = new QuickUriGroupingRule(); + Executors.newSingleThreadScheduledExecutor() + .scheduleWithFixedDelay( + new RunnableWithExceptionProtection( + () -> { + if (aiPipelineExecutionCounter.incrementAndGet() % 25 == 0) { Review Comment: ```suggestion if (aiPipelineExecutionCounter.incrementAndGet() % 30 == 0) { ``` ########## oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java: ########## @@ -18,32 +18,91 @@ package org.apache.skywalking.oap.server.core.config.group; +import io.vavr.Tuple2; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.library.util.StringFormatGroup; +import org.apache.skywalking.oap.server.ai.pipeline.services.api.HttpUriPattern; +import org.apache.skywalking.oap.server.ai.pipeline.services.api.HttpUriRecognition; import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRule4Openapi; +import org.apache.skywalking.oap.server.core.config.group.uri.quickmatch.QuickUriGroupingRule; +import org.apache.skywalking.oap.server.core.query.MetadataQueryService; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.RunnableWithExceptionProtection; +import org.apache.skywalking.oap.server.library.util.StringFormatGroup; @Slf4j public class EndpointNameGrouping { @Setter - private volatile EndpointGroupingRule endpointGroupingRule; + private volatile QuickUriGroupingRule endpointGroupingRule; @Setter private volatile EndpointGroupingRule4Openapi endpointGroupingRule4Openapi; + @Setter + private volatile QuickUriGroupingRule quickUriGroupingRule; + /** + * Cache the HTTP URIs which are not formatted by the rules per service. + * Level one map key is service name, the value is a map of HTTP URI and its count. + * Multiple matches will be counted, because it is the pattern that the endpoint is already formatted, + * or doesn't need to be formatted. + * The repeatable URI is a pattern already. + */ + private ConcurrentHashMap<String, ConcurrentHashMap<String, AtomicInteger>> cachedHttpUris = new ConcurrentHashMap<>(); + private final AtomicInteger aiPipelineExecutionCounter = new AtomicInteger(0); + /** + * The max number of HTTP URIs per service for further URI pattern recognition. + */ + private int maxHttpUrisNumberPerService = 3000; - public String format(String serviceName, String endpointName) { - String formattedName = endpointName; + /** + * Format the endpoint name according to the API patterns. + * + * @param serviceName service name + * @param endpointName endpoint name to be formatted. + * @return Tuple2 where the first element is the formatted name and the second element is a boolean + * represented the endpoint name is formatted or not. + */ + public Tuple2<String, Boolean> format(String serviceName, String endpointName) { + Tuple2<String, Boolean> formattedName = new Tuple2<>(endpointName, Boolean.FALSE); if (endpointGroupingRule4Openapi != null) { - formattedName = formatByOpenapi(serviceName, formattedName); + formattedName = formatByOpenapi(serviceName, endpointName); + } + + if (!formattedName._2() && endpointGroupingRule != null) { + formattedName = formatByCustom(serviceName, endpointName); + } + + if (!formattedName._2() && quickUriGroupingRule != null) { + formattedName = formatByQuickUriPattern(serviceName, endpointName); } - if (endpointGroupingRule != null) { - formattedName = formatByCustom(serviceName, formattedName); + if (!formattedName._2()) { + // Only URI starts with '/' will be cached and formatted later. + if (endpointName.indexOf("/") > -1) { Review Comment: `endpointName.indexOf("/") > -1` doesn't match the comment `Only URI starts with '/' will be cached and formatted later.` ```suggestion // Only URI starts with '/' will be cached and formatted later. if (endpointName.startsWith("/")) { ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
