This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new e6f1e3b4c6 Allow using a dedicated port for ALS receiver (#11749)
e6f1e3b4c6 is described below
commit e6f1e3b4c68d4e09fa08a6e5c4deea0dd704f6ad
Author: kezhenxu94 <[email protected]>
AuthorDate: Thu Jan 11 15:05:47 2024 +0800
Allow using a dedicated port for ALS receiver (#11749)
---
docs/en/changes/changes.md | 1 +
docs/en/setup/backend/backend-load-balancer.md | 48 +++++++++++++++-
docs/en/setup/backend/configuration-vocabulary.md | 8 +++
.../receiver/envoy/EnvoyMetricReceiverConfig.java | 19 +++++++
.../envoy/EnvoyMetricReceiverProvider.java | 64 ++++++++++++++++++++--
.../src/main/resources/application.yml | 9 +++
6 files changed, 143 insertions(+), 6 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 16d6d3cc6b..477673fe03 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -17,6 +17,7 @@
* [Break Change] Change the configuration field of `ui_template` and `ui_menu`
in Elasticsearch storage from **keyword** type to **text**.
* Support Service Hierarchy auto matching.
* Add `namespace` suffix for `K8S_SERVICE_NAME_RULE/ISTIO_SERVICE_NAME_RULE`
and `metadata-service-mapping.yaml` as default.
+* Allow using a dedicated port for ALS receiver.
#### UI
diff --git a/docs/en/setup/backend/backend-load-balancer.md
b/docs/en/setup/backend/backend-load-balancer.md
index 4773bb47dc..f496a7d92b 100644
--- a/docs/en/setup/backend/backend-load-balancer.md
+++ b/docs/en/setup/backend/backend-load-balancer.md
@@ -33,7 +33,7 @@ each of the OAP instance can have similar amount of gRPC
connections.
Before that, you need to calculate the number of connections for each OAP
instance as follows:
-```
+```shell
NUMBER_OF_SERVICE_PODS=<the-number-of-service-pods-that-are-monitored-by-skywalking>
# Each service Pod has 2 connections to OAP
@@ -75,3 +75,49 @@ spec:
app: oap
EOF
```
+
+By this approach, we can limit the connections to port 11800 per OAP instance,
but there
+is another corner case when the amount of service Pods are huge. Because the
limiting is
+on connection level, and each service Pod has 2 connections to OAP port 11800,
one for
+Envoy ALS to send access log, the other one for Envoy metrics, and because the
traffic
+of the 2 connections can vary very much, if the number of service Pods is
large enough,
+an extreme case might happen that one OAP instance is serving all Envoy
metrics connections
+and the other OAP instance is serving all Envoy ALS connections, which in turn
might be
+unbalanced again, to solve this, we can split the ALS connections to a
dedicated port,
+and limit the connections to that port only.
+
+You can set the environment variable `SW_ALS_GRPC_PORT` to a port number other
than `0`
+when deploying skywalking, and limit connections to that port only in the
`EnvoyFilter`:
+
+```shell
+export SW_ALS_GRPC_PORT=11802
+
+kubectl -n $SKYWALKING_NAMESPACE apply -f - <<EOF
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+ name: oap-limit-connections
+ namespace: istio-system
+spec:
+ configPatches:
+ - applyTo: NETWORK_FILTER
+ match:
+ context: ANY
+ listener:
+ filterChain:
+ filter:
+ name: envoy.filters.network.http_connection_manager
+ portNumber: $SW_ALS_GRPC_PORT
+ patch:
+ operation: INSERT_BEFORE
+ value:
+ name: envoy.filters.network.ConnectionLimit
+ typed_config:
+ '@type':
type.googleapis.com/envoy.extensions.filters.network.connection_limit.v3.ConnectionLimit
+ max_connections: $NUMBER_OF_CONNECTIONS_PER_OAP
+ stat_prefix: envoy_filters_network_connection_limit
+ workloadSelector:
+ labels:
+ app: oap
+EOF
+```
diff --git a/docs/en/setup/backend/configuration-vocabulary.md
b/docs/en/setup/backend/configuration-vocabulary.md
index 88cc4c4b4b..25dea0650c 100644
--- a/docs/en/setup/backend/configuration-vocabulary.md
+++ b/docs/en/setup/backend/configuration-vocabulary.md
@@ -192,6 +192,14 @@ The Configuration Vocabulary lists all available
configurations provided by `app
| - | - | k8sServiceNameRule
| `k8sServiceNameRule`
allows you to customize the service name in ALS via Kubernetes metadata. The
available variables are `pod` and `service`. E.g. you can use
`${service.metadata.name}-${pod.metadata.labels.version}` to append the version
number to the service name. Note that [...]
| - | - | istioServiceNameRule
| `istioServiceNameRule`
allows you to customize the service name in ALS via Kubernetes metadata. The
available variables are `serviceEntry`. E.g. you can use
`${serviceEntry.metadata.name}-${serviceEntry.metadata.labels.version}` to
append the version number to the service name [...]
| - | - | istioServiceEntryIgnoredNamespaces
| When looking up service
informations from the Istio ServiceEntries, some of the ServiceEntries might be
created in several namespaces automatically by some components, and OAP will
randomly pick one of them to build the service name, users can use this config
to exclude Servic [...]
+| - | - | gRPCHost
| Binding IP of gRPC
service for Envoy access log service.
[...]
+| - | - | gRPCPort
| Binding port of gRPC
service for Envoy access log service.
[...]
+| - | - | gRPCThreadPoolSize
| Pool size of gRPC
server.
[...]
+| - | - | gRPCSslEnabled
| Activates SSL for gRPC
services.
[...]
+| - | - | gRPCSslKeyPath
| File path of gRPC SSL
key.
[...]
+| - | - | gRPCSslCertChainPath
| File path of gRPC SSL
cert chain.
[...]
+| - | - | maxConcurrentCallsPerConnection
| The maximum number of
concurrent calls permitted for each incoming connection. Defaults to no limit.
[...]
+| - | - | maxMessageSize
| Sets the maximum
message size allowed to be received on the server. Empty means 4 MiB.
[...]
| receiver-otel | default | A receiver for analyzing metrics
data from OpenTelemetry.
| -
[...]
| - | - | enabledHandlers
| Enabled handlers for
otel.
[...]
| - | - | enabledOtelMetricsRules
| Enabled metric rules
for OTLP handler.
[...]
diff --git
a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverConfig.java
b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverConfig.java
index 0ef2d77429..5ae7cee716 100644
---
a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverConfig.java
+++
b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverConfig.java
@@ -45,6 +45,25 @@ public class EnvoyMetricReceiverConfig extends ModuleConfig {
private String istioServiceNameRule;
private String istioServiceEntryIgnoredNamespaces;
+ @Getter
+ private String gRPCHost;
+ @Getter
+ private int gRPCPort;
+ @Getter
+ private int maxConcurrentCallsPerConnection;
+ @Getter
+ private int maxMessageSize;
+ @Getter
+ private int gRPCThreadPoolSize;
+ @Getter
+ private boolean gRPCSslEnabled = false;
+ @Getter
+ private String gRPCSslKeyPath;
+ @Getter
+ private String gRPCSslCertChainPath;
+ @Getter
+ private String gRPCSslTrustedCAsPath;
+
private final ServiceMetaInfoFactory serviceMetaInfoFactory = new
ServiceMetaInfoFactoryImpl();
@Getter
private final ClusterManagerMetricsAdapter clusterManagerMetricsAdapter =
new ClusterManagerMetricsAdapter(this);
diff --git
a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverProvider.java
b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverProvider.java
index d07998748d..765b984315 100644
---
a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverProvider.java
+++
b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverProvider.java
@@ -18,20 +18,29 @@
package org.apache.skywalking.oap.server.receiver.envoy;
+import org.apache.logging.log4j.util.Strings;
import org.apache.skywalking.aop.server.receiver.mesh.MeshReceiverModule;
import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.RunningMode;
import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService;
import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister;
+import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegisterImpl;
import org.apache.skywalking.oap.server.library.module.ModuleDefine;
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import
org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException;
+import org.apache.skywalking.oap.server.library.server.ServerException;
+import org.apache.skywalking.oap.server.library.server.grpc.GRPCServer;
import org.apache.skywalking.oap.server.receiver.envoy.als.mx.FieldsHelper;
import
org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule;
import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
+import java.util.Objects;
+
public class EnvoyMetricReceiverProvider extends ModuleProvider {
protected EnvoyMetricReceiverConfig config;
+ protected GRPCServer grpcServer;
+ protected GRPCHandlerRegister receiverGRPCHandlerRegister;
protected String fieldMappingFile = "metadata-service-mapping.yaml";
@@ -67,6 +76,35 @@ public class EnvoyMetricReceiverProvider extends
ModuleProvider {
} catch (final Exception e) {
throw new ModuleStartException("Failed to load
metadata-service-mapping.yaml", e);
}
+
+ if (config.getGRPCPort() != 0 && !RunningMode.isInitMode()) {
+ if (config.isGRPCSslEnabled()) {
+ grpcServer = new GRPCServer(
+ Strings.isBlank(config.getGRPCHost()) ? "0.0.0.0" :
config.getGRPCHost(),
+ config.getGRPCPort(),
+ config.getGRPCSslCertChainPath(),
+ config.getGRPCSslKeyPath(),
+ config.getGRPCSslTrustedCAsPath()
+ );
+ } else {
+ grpcServer = new GRPCServer(
+ Strings.isBlank(config.getGRPCHost()) ? "0.0.0.0" :
config.getGRPCHost(),
+ config.getGRPCPort()
+ );
+ }
+ if (config.getMaxMessageSize() > 0) {
+ grpcServer.setMaxMessageSize(config.getMaxMessageSize());
+ }
+ if (config.getMaxConcurrentCallsPerConnection() > 0) {
+
grpcServer.setMaxConcurrentCallsPerConnection(config.getMaxConcurrentCallsPerConnection());
+ }
+ if (config.getGRPCThreadPoolSize() > 0) {
+ grpcServer.setThreadPoolSize(config.getGRPCThreadPoolSize());
+ }
+ grpcServer.initialize();
+
+ this.receiverGRPCHandlerRegister = new
GRPCHandlerRegisterImpl(grpcServer);
+ }
}
@Override
@@ -78,15 +116,25 @@ public class EnvoyMetricReceiverProvider extends
ModuleProvider {
.load(TCPOALDefine.INSTANCE);
}
- GRPCHandlerRegister service =
getManager().find(SharingServerModule.NAME)
- .provider()
-
.getService(GRPCHandlerRegister.class);
if (config.isAcceptMetricsService()) {
final MetricServiceGRPCHandler handler = new
MetricServiceGRPCHandler(getManager(), config);
+ // Always use the sharing gRPC server to accept metrics
connections.
+ final var service = getManager()
+ .find(SharingServerModule.NAME)
+ .provider()
+ .getService(GRPCHandlerRegister.class);
service.addHandler(handler);
service.addHandler(new MetricServiceGRPCHandlerV3(handler));
}
- final AccessLogServiceGRPCHandler handler = new
AccessLogServiceGRPCHandler(getManager(), config);
+
+ final var service =
+ Objects.nonNull(receiverGRPCHandlerRegister) ?
+ receiverGRPCHandlerRegister :
+ getManager()
+ .find(SharingServerModule.NAME)
+ .provider()
+ .getService(GRPCHandlerRegister.class);
+ final var handler = new AccessLogServiceGRPCHandler(getManager(),
config);
service.addHandler(handler);
service.addHandler(new AccessLogServiceGRPCHandlerV3(handler));
service.addHandler(new
SatelliteAccessLogServiceGRPCHandlerV3(handler));
@@ -94,7 +142,13 @@ public class EnvoyMetricReceiverProvider extends
ModuleProvider {
@Override
public void notifyAfterCompleted() throws ServiceNotProvidedException,
ModuleStartException {
-
+ try {
+ if (Objects.nonNull(grpcServer) && !RunningMode.isInitMode()) {
+ grpcServer.start();
+ }
+ } catch (ServerException e) {
+ throw new ModuleStartException(e.getMessage(), e);
+ }
}
@Override
diff --git a/oap-server/server-starter/src/main/resources/application.yml
b/oap-server/server-starter/src/main/resources/application.yml
index f47a325e62..a0f105b37d 100644
--- a/oap-server/server-starter/src/main/resources/application.yml
+++ b/oap-server/server-starter/src/main/resources/application.yml
@@ -322,6 +322,15 @@ envoy-metric:
# service name, users can use this config to exclude ServiceEntries that
# they don't want to be used. Comma separated.
istioServiceEntryIgnoredNamespaces:
${SW_ISTIO_SERVICE_ENTRY_IGNORED_NAMESPACES:""}
+ gRPCHost: ${SW_ALS_GRPC_HOST:0.0.0.0}
+ gRPCPort: ${SW_ALS_GRPC_PORT:0}
+ maxConcurrentCallsPerConnection: ${SW_ALS_GRPC_MAX_CONCURRENT_CALL:0}
+ maxMessageSize: ${SW_ALS_GRPC_MAX_MESSAGE_SIZE:0}
+ gRPCThreadPoolSize: ${SW_ALS_GRPC_THREAD_POOL_SIZE:0}
+ gRPCSslEnabled: ${SW_ALS_GRPC_SSL_ENABLED:false}
+ gRPCSslKeyPath: ${SW_ALS_GRPC_SSL_KEY_PATH:""}
+ gRPCSslCertChainPath: ${SW_ALS_GRPC_SSL_CERT_CHAIN_PATH:""}
+ gRPCSslTrustedCAsPath: ${SW_ALS_GRPC_SSL_TRUSTED_CAS_PATH:""}
kafka-fetcher:
selector: ${SW_KAFKA_FETCHER:-}
