This is an automated email from the ASF dual-hosted git repository. wusheng pushed a commit to branch sync-period in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 53b27f76e35360b6e569fd057f07bd3d7461ec75 Author: Wu Sheng <[email protected]> AuthorDate: Mon Jun 26 15:34:47 2023 +0800 Control the period of training and sync HTTP URI recognition patterns --- docs/en/changes/changes.md | 9 +++++++-- docs/en/setup/backend/configuration-vocabulary.md | 2 ++ .../skywalking/oap/server/core/CoreModuleConfig.java | 13 +++++++++++++ .../skywalking/oap/server/core/CoreModuleProvider.java | 2 ++ .../oap/server/core/config/group/EndpointNameGrouping.java | 14 ++++++++------ .../server-starter/src/main/resources/application.yml | 4 ++++ 6 files changed, 36 insertions(+), 8 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 5bd587794b..07d9cd7463 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -15,15 +15,20 @@ * Fix AI Pipeline uri caching NullPointer and IllegalArgument Exceptions. * Fix `NPE` in metrics query when the metric is not exist. * Remove E2E tests for Istio < 1.15, ElasticSearch < 7.16.3, they might still work but are not supported as planed. -* Scroll all results in ElasticSearch storage and refactor scrolling logics, including Service, Instance, Endpoint, Process, etc. +* Scroll all results in ElasticSearch storage and refactor scrolling logics, including Service, Instance, Endpoint, + Process, etc. * Improve Kubernetes coordinator to remove `Terminating` OAP Pods in cluster. +* Support `SW_CORE_SYNC_PERIOD_HTTP_URI_RECOGNITION_PATTERN` and `SW_CORE_TRAINING_PERIOD_HTTP_URI_RECOGNITION_PATTERN` + to control the period of training and sync HTTP URI recognition patterns. And shorten the default period to 10s for + sync and 60s for training. #### UI + * Fix metric name `browser_app_error_rate` in `Browser-Root` dashboard. #### Documentation + * Add Go agent into the server agent documentation. * Add data unit description in the configuration of continuous profiling policy. - All issues and pull requests are [here](https://github.com/apache/skywalking/milestone/181?closed=1) diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index b5e06c94c6..d27c07f5fb 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -83,6 +83,8 @@ The Configuration Vocabulary lists all available configurations provided by `app | - | - | password | Nacos Auth password. [...] | - | - | accessKey | Nacos Auth accessKey. [...] | - | - | secretKey | Nacos Auth secretKey. [...] +| - | - | syncPeriodHttpUriRecognitionPattern | The period of HTTP URI recognition pattern synchronization (in seconds). [...] +| - | - | trainingPeriodHttpUriRecognitionPattern | The period of HTTP URI recognition pattern training (in seconds). [...] | - | - | maxHttpUrisNumberPerService | The maximum number of HTTP URIs per service. [...] | storage | elasticsearch | - | ElasticSearch (and OpenSearch) storage implementation. [...] | - | - | namespace | Prefix of indexes created and used by SkyWalking. [...] diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java index 2f7f74fdb3..40d036e9fa 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java @@ -194,8 +194,21 @@ public class CoreModuleConfig extends ModuleConfig { */ private int httpMaxRequestHeaderSize = 8192; + /** + * The period of HTTP URI pattern recognition. Unit is second. + * @since 9.5.0 + */ + private int syncPeriodHttpUriRecognitionPattern = 10; + + /** + * The training period of HTTP URI pattern recognition. Unit is second. + * @since 9.5.0 + */ + private int trainingPeriodHttpUriRecognitionPattern = 60; + /** * The max number of HTTP URIs per service for further URI pattern recognition. + * @since 9.5.0 */ private int maxHttpUrisNumberPerService = 3000; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index 0223213c75..8516ab2730 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -357,6 +357,8 @@ public class CoreModuleProvider extends ModuleProvider { .provider() .getService(HttpUriRecognition.class), getService(MetadataQueryService.class), + moduleConfig.getSyncPeriodHttpUriRecognitionPattern(), + moduleConfig.getTrainingPeriodHttpUriRecognitionPattern(), moduleConfig.getMaxHttpUrisNumberPerService() ); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java index 8ac88b391e..b3ccc3a671 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java @@ -91,7 +91,7 @@ public class EndpointNameGrouping { formattedName = formatByQuickUriPattern(serviceName, endpointName); ConcurrentHashMap<String, ArrayBlockingQueue<String>> svrHttpUris = - cachedHttpUris.computeIfAbsent(serviceName, k -> new ConcurrentHashMap<>()); + cachedHttpUris.computeIfAbsent(serviceName, k -> new ConcurrentHashMap<>()); // Only cache first N (determined by maxHttpUrisNumberPerService) URIs per 30 mins. if (svrHttpUris.size() < maxHttpUrisNumberPerService) { @@ -100,7 +100,7 @@ public class EndpointNameGrouping { // code may accidentally retreive the size 1 queue created by unformatted endpoint // The queue size is 10, which means only cache the first 10 formatted names. final ArrayBlockingQueue<String> formattedURIs = svrHttpUris.computeIfAbsent( - formattedName._1(), k -> new ArrayBlockingQueue<>(10)); + formattedName._1(), k -> new ArrayBlockingQueue<>(10)); if (formattedURIs.size() < 10) { // Try to push the raw URI as a candidate of formatted name. formattedURIs.offer(endpointName); @@ -160,6 +160,8 @@ public class EndpointNameGrouping { public void startHttpUriRecognitionSvr(final HttpUriRecognition httpUriRecognitionSvr, final MetadataQueryService metadataQueryService, + int syncPeriodHttpUriRecognitionPattern, + int trainingPeriodHttpUriRecognitionPattern, int maxEndpointCandidatePerSvr) { this.maxHttpUrisNumberPerService = maxEndpointCandidatePerSvr; if (!httpUriRecognitionSvr.isInitialized()) { @@ -170,8 +172,8 @@ public class EndpointNameGrouping { .scheduleWithFixedDelay( new RunnableWithExceptionProtection( () -> { - if (aiPipelineExecutionCounter.incrementAndGet() % 30 == 0) { - // Send the cached URIs to the recognition server per 30 mins to build new patterns. + if (aiPipelineExecutionCounter.incrementAndGet() % trainingPeriodHttpUriRecognitionPattern == 0) { + // Send the cached URIs to the recognition server to build new patterns. cachedHttpUris.forEach((serviceName, httpUris) -> { final List<HttpUriRecognition.HTTPUri> candidates4UriPatterns = new ArrayList<>( 3000); @@ -192,7 +194,7 @@ public class EndpointNameGrouping { httpUris.clear(); httpUriRecognitionSvr.feedRawData(serviceName, candidates4UriPatterns); }); - } else { + } else if (aiPipelineExecutionCounter.incrementAndGet() % syncPeriodHttpUriRecognitionPattern == 0) { // Sync with the recognition server per 1 min to get the latest patterns. try { metadataQueryService.listServices(null, null).forEach( @@ -216,7 +218,7 @@ public class EndpointNameGrouping { } }, t -> log.error("Fail to recognize URI patterns.", t) - ), 1, 1, TimeUnit.MINUTES + ), 60, 1, TimeUnit.SECONDS ); } diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index fbaed52194..a4b334ca66 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -129,6 +129,10 @@ core: prepareThreads: ${SW_CORE_PREPARE_THREADS:2} # Turn it on then automatically grouping endpoint by the given OpenAPI definitions. enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPENAPI:true} + # The period of HTTP URI pattern recognition. Unit is second. + syncPeriodHttpUriRecognitionPattern: ${SW_CORE_SYNC_PERIOD_HTTP_URI_RECOGNITION_PATTERN:10} + # The training period of HTTP URI pattern recognition. Unit is second. + trainingPeriodHttpUriRecognitionPattern: ${SW_CORE_TRAINING_PERIOD_HTTP_URI_RECOGNITION_PATTERN:60} # The max number of HTTP URIs per service for further URI pattern recognition. maxHttpUrisNumberPerService: ${SW_CORE_MAX_HTTP_URIS_NUMBER_PER_SVR:3000} storage:
