This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 77da556c9e Control the period of training and sync HTTP URI
recognition patterns (#10997)
77da556c9e is described below
commit 77da556c9e3f1adcde0b996ac926a53daad9d2ae
Author: 吴晟 Wu Sheng <[email protected]>
AuthorDate: Mon Jun 26 19:02:56 2023 +0800
Control the period of training and sync HTTP URI recognition patterns
(#10997)
---
docs/en/changes/changes.md | 9 +++++++--
docs/en/setup/backend/configuration-vocabulary.md | 2 ++
.../skywalking/oap/server/core/CoreModuleConfig.java | 13 +++++++++++++
.../skywalking/oap/server/core/CoreModuleProvider.java | 2 ++
.../server/core/config/group/EndpointNameGrouping.java | 15 +++++++++------
.../server-starter/src/main/resources/application.yml | 4 ++++
6 files changed, 37 insertions(+), 8 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 5bd587794b..07d9cd7463 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -15,15 +15,20 @@
* Fix AI Pipeline uri caching NullPointer and IllegalArgument Exceptions.
* Fix `NPE` in metrics query when the metric is not exist.
* Remove E2E tests for Istio < 1.15, ElasticSearch < 7.16.3, they might still
work but are not supported as planed.
-* Scroll all results in ElasticSearch storage and refactor scrolling logics,
including Service, Instance, Endpoint, Process, etc.
+* Scroll all results in ElasticSearch storage and refactor scrolling logics,
including Service, Instance, Endpoint,
+ Process, etc.
* Improve Kubernetes coordinator to remove `Terminating` OAP Pods in cluster.
+* Support `SW_CORE_SYNC_PERIOD_HTTP_URI_RECOGNITION_PATTERN` and
`SW_CORE_TRAINING_PERIOD_HTTP_URI_RECOGNITION_PATTERN`
+ to control the period of training and sync HTTP URI recognition patterns.
And shorten the default period to 10s for
+ sync and 60s for training.
#### UI
+
* Fix metric name `browser_app_error_rate` in `Browser-Root` dashboard.
#### Documentation
+
* Add Go agent into the server agent documentation.
* Add data unit description in the configuration of continuous profiling
policy.
-
All issues and pull requests are
[here](https://github.com/apache/skywalking/milestone/181?closed=1)
diff --git a/docs/en/setup/backend/configuration-vocabulary.md
b/docs/en/setup/backend/configuration-vocabulary.md
index b5e06c94c6..d27c07f5fb 100644
--- a/docs/en/setup/backend/configuration-vocabulary.md
+++ b/docs/en/setup/backend/configuration-vocabulary.md
@@ -83,6 +83,8 @@ The Configuration Vocabulary lists all available
configurations provided by `app
| - | - | password
| Nacos Auth password.
[...]
| - | - | accessKey
| Nacos Auth accessKey.
[...]
| - | - | secretKey
| Nacos Auth secretKey.
[...]
+| - | - |
syncPeriodHttpUriRecognitionPattern
| The period of HTTP URI recognition pattern synchronization (in
seconds).
[...]
+| - | - |
trainingPeriodHttpUriRecognitionPattern
| The period of HTTP URI recognition pattern training (in seconds).
[...]
| - | - | maxHttpUrisNumberPerService
| The maximum number of
HTTP URIs per service.
[...]
| storage | elasticsearch | -
| ElasticSearch (and
OpenSearch) storage implementation.
[...]
| - | - | namespace
| Prefix of indexes
created and used by SkyWalking.
[...]
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
index 2f7f74fdb3..40d036e9fa 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
@@ -194,8 +194,21 @@ public class CoreModuleConfig extends ModuleConfig {
*/
private int httpMaxRequestHeaderSize = 8192;
+ /**
+ * The period of HTTP URI pattern recognition. Unit is second.
+ * @since 9.5.0
+ */
+ private int syncPeriodHttpUriRecognitionPattern = 10;
+
+ /**
+ * The training period of HTTP URI pattern recognition. Unit is second.
+ * @since 9.5.0
+ */
+ private int trainingPeriodHttpUriRecognitionPattern = 60;
+
/**
* The max number of HTTP URIs per service for further URI pattern
recognition.
+ * @since 9.5.0
*/
private int maxHttpUrisNumberPerService = 3000;
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
index 0223213c75..8516ab2730 100755
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
@@ -357,6 +357,8 @@ public class CoreModuleProvider extends ModuleProvider {
.provider()
.getService(HttpUriRecognition.class),
getService(MetadataQueryService.class),
+ moduleConfig.getSyncPeriodHttpUriRecognitionPattern(),
+ moduleConfig.getTrainingPeriodHttpUriRecognitionPattern(),
moduleConfig.getMaxHttpUrisNumberPerService()
);
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
index 8ac88b391e..c122db7fae 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
@@ -91,7 +91,7 @@ public class EndpointNameGrouping {
formattedName = formatByQuickUriPattern(serviceName, endpointName);
ConcurrentHashMap<String, ArrayBlockingQueue<String>> svrHttpUris =
- cachedHttpUris.computeIfAbsent(serviceName, k -> new
ConcurrentHashMap<>());
+ cachedHttpUris.computeIfAbsent(serviceName, k -> new
ConcurrentHashMap<>());
// Only cache first N (determined by maxHttpUrisNumberPerService)
URIs per 30 mins.
if (svrHttpUris.size() < maxHttpUrisNumberPerService) {
@@ -100,7 +100,7 @@ public class EndpointNameGrouping {
// code may accidentally retreive the size 1 queue created
by unformatted endpoint
// The queue size is 10, which means only cache the first
10 formatted names.
final ArrayBlockingQueue<String> formattedURIs =
svrHttpUris.computeIfAbsent(
- formattedName._1(), k -> new
ArrayBlockingQueue<>(10));
+ formattedName._1(), k -> new ArrayBlockingQueue<>(10));
if (formattedURIs.size() < 10) {
// Try to push the raw URI as a candidate of formatted
name.
formattedURIs.offer(endpointName);
@@ -160,6 +160,8 @@ public class EndpointNameGrouping {
public void startHttpUriRecognitionSvr(final HttpUriRecognition
httpUriRecognitionSvr,
final MetadataQueryService
metadataQueryService,
+ int
syncPeriodHttpUriRecognitionPattern,
+ int
trainingPeriodHttpUriRecognitionPattern,
int maxEndpointCandidatePerSvr) {
this.maxHttpUrisNumberPerService = maxEndpointCandidatePerSvr;
if (!httpUriRecognitionSvr.isInitialized()) {
@@ -170,8 +172,8 @@ public class EndpointNameGrouping {
.scheduleWithFixedDelay(
new RunnableWithExceptionProtection(
() -> {
- if (aiPipelineExecutionCounter.incrementAndGet()
% 30 == 0) {
- // Send the cached URIs to the recognition
server per 30 mins to build new patterns.
+ if (aiPipelineExecutionCounter.incrementAndGet()
% trainingPeriodHttpUriRecognitionPattern == 0) {
+ // Send the cached URIs to the recognition
server to build new patterns.
cachedHttpUris.forEach((serviceName,
httpUris) -> {
final List<HttpUriRecognition.HTTPUri>
candidates4UriPatterns = new ArrayList<>(
3000);
@@ -192,7 +194,8 @@ public class EndpointNameGrouping {
httpUris.clear();
httpUriRecognitionSvr.feedRawData(serviceName, candidates4UriPatterns);
});
- } else {
+ }
+ if (aiPipelineExecutionCounter.incrementAndGet()
% syncPeriodHttpUriRecognitionPattern == 0) {
// Sync with the recognition server per 1 min
to get the latest patterns.
try {
metadataQueryService.listServices(null,
null).forEach(
@@ -216,7 +219,7 @@ public class EndpointNameGrouping {
}
},
t -> log.error("Fail to recognize URI patterns.", t)
- ), 1, 1, TimeUnit.MINUTES
+ ), 60, 1, TimeUnit.SECONDS
);
}
diff --git a/oap-server/server-starter/src/main/resources/application.yml
b/oap-server/server-starter/src/main/resources/application.yml
index fbaed52194..a4b334ca66 100644
--- a/oap-server/server-starter/src/main/resources/application.yml
+++ b/oap-server/server-starter/src/main/resources/application.yml
@@ -129,6 +129,10 @@ core:
prepareThreads: ${SW_CORE_PREPARE_THREADS:2}
# Turn it on then automatically grouping endpoint by the given OpenAPI
definitions.
enableEndpointNameGroupingByOpenapi:
${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPENAPI:true}
+ # The period of HTTP URI pattern recognition. Unit is second.
+ syncPeriodHttpUriRecognitionPattern:
${SW_CORE_SYNC_PERIOD_HTTP_URI_RECOGNITION_PATTERN:10}
+ # The training period of HTTP URI pattern recognition. Unit is second.
+ trainingPeriodHttpUriRecognitionPattern:
${SW_CORE_TRAINING_PERIOD_HTTP_URI_RECOGNITION_PATTERN:60}
# The max number of HTTP URIs per service for further URI pattern
recognition.
maxHttpUrisNumberPerService: ${SW_CORE_MAX_HTTP_URIS_NUMBER_PER_SVR:3000}
storage:
