This is an automated email from the ASF dual-hosted git repository.
mfordjody pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new 94776147 Add egress gateway and adjustment observability (#930)
94776147 is described below
commit 94776147f3ec2ba51fd3e80cb4c45793a2a902fc
Author: mfordjody <[email protected]>
AuthorDate: Mon Jun 29 15:38:16 2026 +0800
Add egress gateway and adjustment observability (#930)
---
cli/cmd/dashboard.go | 38 ++++-
cli/cmd/dashboard_test.go | 30 ++++
.../pkg/bootstrap/proxyless_grpc_controller.go | 2 +-
dubbod/discovery/pkg/bootstrap/server.go | 3 +
.../pkg/config/kube/crdclient/types.gen.go | 51 +++++++
.../config/kube/gateway/deployment_controller.go | 117 ++++++++++++++-
.../kube/gateway/deployment_controller_test.go | 119 +++++++++++++++-
dubbod/discovery/pkg/model/push_context.go | 86 ++++++++++-
dubbod/discovery/pkg/networking/grpcgen/cds.go | 28 ++++
.../discovery/pkg/networking/grpcgen/cds_test.go | 91 ++++++++++++
.../pkg/xds/endpoints/endpoint_builder.go | 30 ++++
.../pkg/xds/endpoints/endpoint_builder_test.go | 20 +++
manifests/charts/dubbod/files/kube-gateway.yaml | 7 +
pkg/config/schema/collections/collections.gen.go | 22 +++
pkg/config/schema/gvk/resources.gen.go | 10 ++
pkg/config/schema/gvr/resources.gen.go | 6 +
pkg/config/schema/kind/resources.gen.go | 5 +
pkg/config/schema/kubeclient/resources.gen.go | 13 ++
pkg/config/schema/kubetypes/resources.gen.go | 2 +
pkg/config/schema/metadata.yaml | 11 ++
samples/addons/grafana.yaml | 157 +++++++++++++++++++++
samples/addons/prometheus.yaml | 51 +++++++
samples/addons/{prometheus.yaml => tracing.yaml} | 112 ++++++---------
23 files changed, 920 insertions(+), 91 deletions(-)
diff --git a/cli/cmd/dashboard.go b/cli/cmd/dashboard.go
index 4705c691..2e062f06 100644
--- a/cli/cmd/dashboard.go
+++ b/cli/cmd/dashboard.go
@@ -88,7 +88,7 @@ func DashboardCmd(ctx cli.Context) *cobra.Command {
}
if args.wait > 0 {
- if err := waitForDashboard(cmd.Context(),
client.Kube(), dashboardNamespace, args.wait); err != nil {
+ if err := waitForDashboard(cmd.Context(),
client.Kube(), dashboardNamespace, dashboardWaitDeploymentNames(files),
args.wait); err != nil {
return err
}
if _, err := fmt.Fprintln(cmd.OutOrStdout(),
"dashboard ready"); err != nil {
@@ -97,11 +97,17 @@ func DashboardCmd(ctx cli.Context) *cobra.Command {
}
_, err = fmt.Fprintf(cmd.OutOrStdout(), "prometheus:
kubectl -n %s port-forward svc/prometheus 9090:9090\ngrafana: kubectl -n %s
port-forward svc/grafana 3000:3000\n", dashboardNamespace, dashboardNamespace)
+ if err != nil {
+ return err
+ }
+ if dashboardHasTracing(files) {
+ _, err = fmt.Fprintf(cmd.OutOrStdout(),
"tracing: kubectl -n %s port-forward svc/tracing 16686:16686\n",
dashboardNamespace)
+ }
return err
},
}
command.Flags().StringVar(&args.manifests, "manifests", args.manifests,
"Dashboard manifest file or samples/addons directory")
- command.Flags().DurationVar(&args.wait, "wait", args.wait, "Maximum
time to wait for Prometheus and Grafana deployments; 0 disables waiting")
+ command.Flags().DurationVar(&args.wait, "wait", args.wait, "Maximum
time to wait for observability deployments; 0 disables waiting")
return command
}
@@ -126,7 +132,29 @@ func dashboardManifestFiles(path string) ([]string, error)
{
if _, err := os.Stat(grafana); err != nil {
return nil, fmt.Errorf("dashboard grafana manifest not found in
%s", path)
}
- return []string{prometheus, grafana}, nil
+ files := []string{prometheus, grafana}
+ tracing := filepath.Join(path, "tracing.yaml")
+ if _, err := os.Stat(tracing); err == nil {
+ files = append(files, tracing)
+ }
+ return files, nil
+}
+
+func dashboardHasTracing(files []string) bool {
+ for _, file := range files {
+ if filepath.Base(file) == "tracing.yaml" {
+ return true
+ }
+ }
+ return false
+}
+
+func dashboardWaitDeploymentNames(files []string) []string {
+ names := []string{"prometheus", "grafana"}
+ if dashboardHasTracing(files) {
+ names = append(names, "tracing")
+ }
+ return names
}
func applyManifestFile(ctx context.Context, client kube.CLIClient, file
string) ([]appliedObject, error) {
@@ -226,8 +254,8 @@ func printAppliedDashboard(writer io.Writer, objects
[]appliedObject) error {
return nil
}
-func waitForDashboard(ctx context.Context, client kubernetes.Interface,
namespace string, timeout time.Duration) error {
- for _, name := range []string{"prometheus", "grafana"} {
+func waitForDashboard(ctx context.Context, client kubernetes.Interface,
namespace string, names []string, timeout time.Duration) error {
+ for _, name := range names {
if err := waitForAvailableDeployment(ctx, client, namespace,
name, timeout); err != nil {
return err
}
diff --git a/cli/cmd/dashboard_test.go b/cli/cmd/dashboard_test.go
index f9105dfb..a087463a 100644
--- a/cli/cmd/dashboard_test.go
+++ b/cli/cmd/dashboard_test.go
@@ -26,6 +26,36 @@ func TestDashboardManifestFiles(t *testing.T) {
if !reflect.DeepEqual(got, want) {
t.Fatalf("dashboardManifestFiles() = %v, want %v", got, want)
}
+ if dashboardHasTracing(got) {
+ t.Fatalf("dashboardHasTracing() = true, want false")
+ }
+ if names := dashboardWaitDeploymentNames(got);
!reflect.DeepEqual(names, []string{"prometheus", "grafana"}) {
+ t.Fatalf("dashboardWaitDeploymentNames() = %v", names)
+ }
+}
+
+func TestDashboardManifestFilesIncludesTracingWhenPresent(t *testing.T) {
+ dir := t.TempDir()
+ for _, name := range []string{"prometheus.yaml", "grafana.yaml",
"tracing.yaml"} {
+ if err := os.WriteFile(filepath.Join(dir, name),
[]byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: "+name+"\n"),
0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ got, err := dashboardManifestFiles(dir)
+ if err != nil {
+ t.Fatalf("dashboardManifestFiles() returned error: %v", err)
+ }
+ want := []string{filepath.Join(dir, "prometheus.yaml"),
filepath.Join(dir, "grafana.yaml"), filepath.Join(dir, "tracing.yaml")}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("dashboardManifestFiles() = %v, want %v", got, want)
+ }
+ if !dashboardHasTracing(got) {
+ t.Fatalf("dashboardHasTracing() = false, want true")
+ }
+ if names := dashboardWaitDeploymentNames(got);
!reflect.DeepEqual(names, []string{"prometheus", "grafana", "tracing"}) {
+ t.Fatalf("dashboardWaitDeploymentNames() = %v", names)
+ }
}
func TestReadManifestObjects(t *testing.T) {
diff --git a/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go
b/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go
index 57b68813..bd2f512b 100644
--- a/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go
+++ b/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go
@@ -238,7 +238,7 @@ func proxylessGRPCRuntimeConfigNeedsUpdate(req
*discoverymodel.PushRequest) bool
}
for cfg := range req.ConfigsUpdated {
switch cfg.Kind {
- case kind.HTTPRoute, kind.CircuitBreakerPolicy,
kind.PeerAuthentication, kind.RequestAuthentication, kind.AuthorizationPolicy,
kind.Service, kind.EndpointSlice, kind.Endpoints, kind.Pod, kind.Namespace:
+ case kind.HTTPRoute, kind.BackendTLSPolicy,
kind.CircuitBreakerPolicy, kind.PeerAuthentication, kind.RequestAuthentication,
kind.AuthorizationPolicy, kind.Service, kind.EndpointSlice, kind.Endpoints,
kind.Pod, kind.Namespace:
return true
}
}
diff --git a/dubbod/discovery/pkg/bootstrap/server.go
b/dubbod/discovery/pkg/bootstrap/server.go
index f17a7827..d904759d 100644
--- a/dubbod/discovery/pkg/bootstrap/server.go
+++ b/dubbod/discovery/pkg/bootstrap/server.go
@@ -511,6 +511,8 @@ func (s *Server) initRegistryEventHandlers() {
configKind = kind.KubernetesGateway
case "HTTPRoute":
configKind = kind.HTTPRoute
+ case "BackendTLSPolicy":
+ configKind = kind.BackendTLSPolicy
case "CircuitBreakerPolicy":
configKind = kind.CircuitBreakerPolicy
default:
@@ -535,6 +537,7 @@ func (s *Server) initRegistryEventHandlers() {
configKind == kind.RequestAuthentication ||
configKind == kind.AuthorizationPolicy ||
configKind == kind.HTTPRoute ||
+ configKind == kind.BackendTLSPolicy ||
configKind == kind.CircuitBreakerPolicy
// Trigger ConfigUpdate to push changes to all connected proxies
diff --git a/dubbod/discovery/pkg/config/kube/crdclient/types.gen.go
b/dubbod/discovery/pkg/config/kube/crdclient/types.gen.go
index c5bf20ef..3d9de0e7 100755
--- a/dubbod/discovery/pkg/config/kube/crdclient/types.gen.go
+++ b/dubbod/discovery/pkg/config/kube/crdclient/types.gen.go
@@ -37,6 +37,11 @@ func create(c kube.Client, cfg config.Config, objMeta
metav1.ObjectMeta) (metav1
ObjectMeta: objMeta,
Spec:
*(cfg.Spec.(*githubcomkdubboapisecurityv1alpha3.AuthorizationPolicy)),
}, metav1.CreateOptions{})
+ case gvk.BackendTLSPolicy:
+ return
c.GatewayAPI().GatewayV1().BackendTLSPolicies(cfg.Namespace).Create(context.TODO(),
&sigsk8siogatewayapiapisv1.BackendTLSPolicy{
+ ObjectMeta: objMeta,
+ Spec:
*(cfg.Spec.(*sigsk8siogatewayapiapisv1.BackendTLSPolicySpec)),
+ }, metav1.CreateOptions{})
case gvk.CircuitBreakerPolicy:
return
c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(cfg.Namespace).Create(context.TODO(),
&apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{
ObjectMeta: objMeta,
@@ -79,6 +84,11 @@ func update(c kube.Client, cfg config.Config, objMeta
metav1.ObjectMeta) (metav1
ObjectMeta: objMeta,
Spec:
*(cfg.Spec.(*githubcomkdubboapisecurityv1alpha3.AuthorizationPolicy)),
}, metav1.UpdateOptions{})
+ case gvk.BackendTLSPolicy:
+ return
c.GatewayAPI().GatewayV1().BackendTLSPolicies(cfg.Namespace).Update(context.TODO(),
&sigsk8siogatewayapiapisv1.BackendTLSPolicy{
+ ObjectMeta: objMeta,
+ Spec:
*(cfg.Spec.(*sigsk8siogatewayapiapisv1.BackendTLSPolicySpec)),
+ }, metav1.UpdateOptions{})
case gvk.CircuitBreakerPolicy:
return
c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(cfg.Namespace).Update(context.TODO(),
&apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{
ObjectMeta: objMeta,
@@ -121,6 +131,11 @@ func updateStatus(c kube.Client, cfg config.Config,
objMeta metav1.ObjectMeta) (
ObjectMeta: objMeta,
Status:
*(cfg.Status.(*githubcomkdubboapimetav1alpha1.DubboStatus)),
}, metav1.UpdateOptions{})
+ case gvk.BackendTLSPolicy:
+ return
c.GatewayAPI().GatewayV1().BackendTLSPolicies(cfg.Namespace).UpdateStatus(context.TODO(),
&sigsk8siogatewayapiapisv1.BackendTLSPolicy{
+ ObjectMeta: objMeta,
+ Status:
*(cfg.Status.(*sigsk8siogatewayapiapisv1.PolicyStatus)),
+ }, metav1.UpdateOptions{})
case gvk.CircuitBreakerPolicy:
return
c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(cfg.Namespace).UpdateStatus(context.TODO(),
&apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{
ObjectMeta: objMeta,
@@ -176,6 +191,21 @@ func patch(c kube.Client, orig config.Config, origMeta
metav1.ObjectMeta, mod co
}
return
c.Dubbo().SecurityV1alpha3().AuthorizationPolicies(orig.Namespace).
Patch(context.TODO(), orig.Name, typ, patchBytes,
metav1.PatchOptions{FieldManager: "pilot-discovery"})
+ case gvk.BackendTLSPolicy:
+ oldRes := &sigsk8siogatewayapiapisv1.BackendTLSPolicy{
+ ObjectMeta: origMeta,
+ Spec:
*(orig.Spec.(*sigsk8siogatewayapiapisv1.BackendTLSPolicySpec)),
+ }
+ modRes := &sigsk8siogatewayapiapisv1.BackendTLSPolicy{
+ ObjectMeta: modMeta,
+ Spec:
*(mod.Spec.(*sigsk8siogatewayapiapisv1.BackendTLSPolicySpec)),
+ }
+ patchBytes, err := genPatchBytes(oldRes, modRes, typ)
+ if err != nil {
+ return nil, err
+ }
+ return
c.GatewayAPI().GatewayV1().BackendTLSPolicies(orig.Namespace).
+ Patch(context.TODO(), orig.Name, typ, patchBytes,
metav1.PatchOptions{FieldManager: "pilot-discovery"})
case gvk.CircuitBreakerPolicy:
oldRes :=
&apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{
ObjectMeta: origMeta,
@@ -279,6 +309,8 @@ func delete(c kube.Client, typ config.GroupVersionKind,
name, namespace string,
switch typ {
case gvk.AuthorizationPolicy:
return
c.Dubbo().SecurityV1alpha3().AuthorizationPolicies(namespace).Delete(context.TODO(),
name, deleteOptions)
+ case gvk.BackendTLSPolicy:
+ return
c.GatewayAPI().GatewayV1().BackendTLSPolicies(namespace).Delete(context.TODO(),
name, deleteOptions)
case gvk.CircuitBreakerPolicy:
return
c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(namespace).Delete(context.TODO(),
name, deleteOptions)
case gvk.GatewayClass:
@@ -316,6 +348,25 @@ var translationMap = map[config.GroupVersionKind]func(r
runtime.Object) config.C
Status: &obj.Status,
}
},
+ gvk.BackendTLSPolicy: func(r runtime.Object) config.Config {
+ obj := r.(*sigsk8siogatewayapiapisv1.BackendTLSPolicy)
+ return config.Config{
+ Meta: config.Meta{
+ GroupVersionKind: gvk.BackendTLSPolicy,
+ Name: obj.Name,
+ Namespace: obj.Namespace,
+ Labels: obj.Labels,
+ Annotations: obj.Annotations,
+ ResourceVersion: obj.ResourceVersion,
+ CreationTimestamp: obj.CreationTimestamp.Time,
+ OwnerReferences: obj.OwnerReferences,
+ UID: string(obj.UID),
+ Generation: obj.Generation,
+ },
+ Spec: &obj.Spec,
+ Status: &obj.Status,
+ }
+ },
gvk.CircuitBreakerPolicy: func(r runtime.Object) config.Config {
obj :=
r.(*apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy)
return config.Config{
diff --git a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go
b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go
index 324e07ec..6b9362ce 100644
--- a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go
+++ b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go
@@ -79,6 +79,7 @@ const (
serviceTargetPortAnnotation = "gateway.dubbo.apache.org/target-port"
serviceNodePortAnnotation = "gateway.dubbo.apache.org/node-port"
xdsAddressAnnotation = "gateway.dubbo.apache.org/xds-address"
+ otelEndpointAnnotation = "gateway.dubbo.apache.org/otel-endpoint"
)
var builtinClasses = getBuiltinClasses()
@@ -124,6 +125,7 @@ type DeploymentController struct {
configMaps kclient.Client[*corev1.ConfigMap]
httpRoutes kclient.Client[*gateway.HTTPRoute]
circuitBreakers kclient.Client[*clientnetworking.CircuitBreakerPolicy]
+ backendTLS kclient.Client[*gateway.BackendTLSPolicy]
namespaces kclient.Client[*corev1.Namespace]
tagWatcher TagWatcher
revision string
@@ -207,6 +209,11 @@ func NewDeploymentController(
dc.services = kclient.NewFiltered[*corev1.Service](client, filter)
dc.services.AddEventHandler(parentHandler)
+ dc.services.AddEventHandler(controllers.ObjectHandler(func(o
controllers.Object) {
+ for _, gw := range dc.gateways.List(metav1.NamespaceAll,
klabels.Everything()) {
+ dc.queue.AddObject(gw)
+ }
+ }))
dc.clients[gvr.Service] = NewUntypedWrapper(dc.services)
dc.deployments = kclient.NewFiltered[*appsv1.Deployment](client, filter)
@@ -235,6 +242,13 @@ func NewDeploymentController(
}
}))
+ dc.backendTLS = kclient.NewFiltered[*gateway.BackendTLSPolicy](client,
filter)
+ dc.backendTLS.AddEventHandler(controllers.ObjectHandler(func(o
controllers.Object) {
+ for _, gw := range dc.gateways.List(metav1.NamespaceAll,
klabels.Everything()) {
+ dc.queue.AddObject(gw)
+ }
+ }))
+
// Namespace is a cluster-scoped resource, use New instead of
NewFiltered
dc.namespaces = kclient.New[*corev1.Namespace](client)
dc.namespaces.AddEventHandler(controllers.ObjectHandler(func(o
controllers.Object) {
@@ -285,6 +299,7 @@ func (d *DeploymentController) Run(stop <-chan struct{}) {
d.configMaps.HasSynced,
d.httpRoutes.HasSynced,
d.circuitBreakers.HasSynced,
+ d.backendTLS.HasSynced,
d.gateways.HasSynced,
d.gatewayClasses.HasSynced,
)
@@ -301,6 +316,7 @@ func (d *DeploymentController) Run(stop <-chan struct{}) {
d.configMaps,
d.httpRoutes,
d.circuitBreakers,
+ d.backendTLS,
d.gateways,
d.gatewayClasses,
)
@@ -386,6 +402,7 @@ func (d *DeploymentController) configureGateway(log
*dubbolog.Logger, gw gateway
SystemNamespace: d.systemNamespace,
ClusterID: string(d.clusterID),
DomainSuffix: d.domainSuffix(),
+ OtelEndpoint:
strings.TrimSpace(gw.Annotations[otelEndpointAnnotation]),
}
log.Infof("desired dxgate deployment=%s/%s gatewayClass=%s
serviceType=%s ports=%s image=%s",
@@ -438,6 +455,7 @@ type TemplateInput struct {
SystemNamespace string
ClusterID string
DomainSuffix string
+ OtelEndpoint string
}
type dxgateBootstrapConfig struct {
@@ -497,10 +515,16 @@ type dxgateWeightedCluster struct {
type dxgateCluster struct {
Name string `json:"name" yaml:"name"`
Endpoints []dxgateEndpoint `json:"endpoints"
yaml:"endpoints"`
+ TLS *dxgateUpstreamTLS `json:"tls,omitempty"
yaml:"tls,omitempty"`
CircuitBreaker *dxgateCircuitBreaker
`json:"circuit_breaker,omitempty" yaml:"circuit_breaker,omitempty"`
Outlier *dxgateOutlierDetection
`json:"outlier_detection,omitempty" yaml:"outlier_detection,omitempty"`
}
+type dxgateUpstreamTLS struct {
+ Mode string `json:"mode" yaml:"mode"`
+ SNI string `json:"sni,omitempty" yaml:"sni,omitempty"`
+}
+
type dxgateCircuitBreaker struct {
MaxConnections int32 `json:"max_connections,omitempty"
yaml:"max_connections,omitempty"`
HTTP1MaxPendingRequests int32
`json:"http1_max_pending_requests,omitempty"
yaml:"http1_max_pending_requests,omitempty"`
@@ -601,7 +625,9 @@ func formatGatewayServicePorts(ports []corev1.ServicePort)
string {
func (d *DeploymentController) buildDxgateRuntimeConfig(gw gateway.Gateway)
(string, string, error) {
routes := d.httpRoutes.List(metav1.NamespaceAll, klabels.Everything())
- return buildDxgateRuntimeConfig(gw, routes,
d.circuitBreakerPolicyConfigs(), d.domainSuffix())
+ services := d.services.List(metav1.NamespaceAll, klabels.Everything())
+ backendTLS := d.backendTLSPolicies()
+ return buildDxgateRuntimeConfig(gw, routes, services, backendTLS,
d.circuitBreakerPolicyConfigs(), d.domainSuffix())
}
func (d *DeploymentController) circuitBreakerPolicyConfigs() []config.Config {
@@ -634,11 +660,20 @@ func (d *DeploymentController)
circuitBreakerPolicyConfigs() []config.Config {
return nil
}
-func buildDxgateRuntimeConfig(gw gateway.Gateway, routes []*gateway.HTTPRoute,
policies []config.Config, domainSuffix string) (string, string, error) {
+func (d *DeploymentController) backendTLSPolicies()
[]*gateway.BackendTLSPolicy {
+ if d.backendTLS != nil {
+ return d.backendTLS.List(metav1.NamespaceAll,
klabels.Everything())
+ }
+ return nil
+}
+
+func buildDxgateRuntimeConfig(gw gateway.Gateway, routes []*gateway.HTTPRoute,
services []*corev1.Service, backendTLSPolicies []*gateway.BackendTLSPolicy,
policies []config.Config, domainSuffix string) (string, string, error) {
if domainSuffix == "" {
domainSuffix = constants.DefaultClusterLocalDomain
}
circuitBreakers := circuitBreakerPoliciesByTarget(policies)
+ backendTLS := backendTLSPoliciesByTarget(backendTLSPolicies)
+ servicesByKey := servicesByNamespacedName(services)
sort.Slice(routes, func(i, j int) bool {
if routes[i].Namespace != routes[j].Namespace {
@@ -666,7 +701,7 @@ func buildDxgateRuntimeConfig(gw gateway.Gateway, routes
[]*gateway.HTTPRoute, p
if !httpRouteReferencesGateway(hr, &gw) {
continue
}
- vh, clusters := buildDxgateVirtualHost(gw, hr, circuitBreakers,
domainSuffix)
+ vh, clusters := buildDxgateVirtualHost(gw, hr, servicesByKey,
backendTLS, circuitBreakers, domainSuffix)
if len(vh.Routes) == 0 {
continue
}
@@ -700,7 +735,7 @@ func dxgateRuntimeVersion(gw gateway.Gateway, routes
[]*gateway.HTTPRoute) strin
return strings.Join(parts, ";")
}
-func buildDxgateVirtualHost(gw gateway.Gateway, hr *gateway.HTTPRoute,
circuitBreakers map[string]dxgateBackendCircuitBreaker, domainSuffix string)
(dxgateVirtualHost, []dxgateCluster) {
+func buildDxgateVirtualHost(gw gateway.Gateway, hr *gateway.HTTPRoute,
services map[string]*corev1.Service, backendTLS map[string]*dxgateUpstreamTLS,
circuitBreakers map[string]dxgateBackendCircuitBreaker, domainSuffix string)
(dxgateVirtualHost, []dxgateCluster) {
vh := dxgateVirtualHost{
Name: fmt.Sprintf("%s-%s", hr.Namespace, hr.Name),
Domains: dxgateRouteDomains(gw, hr),
@@ -733,7 +768,9 @@ func buildDxgateVirtualHost(gw gateway.Gateway, hr
*gateway.HTTPRoute, circuitBr
}
port := uint16(*backendRef.Port)
clusterName := fmt.Sprintf("%s-%s-%d-%d",
hr.Namespace, hr.Name, ruleIdx, backendIdx)
- policy :=
circuitBreakers[namespacedServiceKey(backendNamespace, string(backendRef.Name))]
+ backendKey :=
namespacedServiceKey(backendNamespace, string(backendRef.Name))
+ policy := circuitBreakers[backendKey]
+ upstreamTLS := backendTLS[backendKey]
route.WeightedClusters =
append(route.WeightedClusters, dxgateWeightedCluster{
Name: clusterName,
@@ -741,11 +778,12 @@ func buildDxgateVirtualHost(gw gateway.Gateway, hr
*gateway.HTTPRoute, circuitBr
})
clusters = append(clusters, dxgateCluster{
Name: clusterName,
+ TLS: upstreamTLS,
CircuitBreaker: policy.CircuitBreaker,
Outlier: policy.Outlier,
Endpoints: []dxgateEndpoint{
{
- Address:
fmt.Sprintf("%s.%s.svc.%s", backendRef.Name, backendNamespace, domainSuffix),
+ Address:
dxgateBackendAddress(backendNamespace, string(backendRef.Name), domainSuffix,
services),
Port: port,
Healthy: true,
},
@@ -760,6 +798,73 @@ func buildDxgateVirtualHost(gw gateway.Gateway, hr
*gateway.HTTPRoute, circuitBr
return vh, clusters
}
+func servicesByNamespacedName(services []*corev1.Service)
map[string]*corev1.Service {
+ out := map[string]*corev1.Service{}
+ for _, svc := range services {
+ if svc == nil {
+ continue
+ }
+ out[namespacedServiceKey(svc.Namespace, svc.Name)] = svc
+ }
+ return out
+}
+
+func dxgateBackendAddress(namespace, name, domainSuffix string, services
map[string]*corev1.Service) string {
+ if svc := services[namespacedServiceKey(namespace, name)]; svc != nil
&& svc.Spec.Type == corev1.ServiceTypeExternalName && svc.Spec.ExternalName !=
"" {
+ return svc.Spec.ExternalName
+ }
+ return fmt.Sprintf("%s.%s.svc.%s", name, namespace, domainSuffix)
+}
+
+func backendTLSPoliciesByTarget(policies []*gateway.BackendTLSPolicy)
map[string]*dxgateUpstreamTLS {
+ out := map[string]*dxgateUpstreamTLS{}
+ sort.Slice(policies, func(i, j int) bool {
+ if
!policies[i].CreationTimestamp.Time.Equal(policies[j].CreationTimestamp.Time) {
+ return
policies[i].CreationTimestamp.Time.Before(policies[j].CreationTimestamp.Time)
+ }
+ if policies[i].Namespace != policies[j].Namespace {
+ return policies[i].Namespace < policies[j].Namespace
+ }
+ return policies[i].Name < policies[j].Name
+ })
+ for _, policy := range policies {
+ if policy == nil || !supportsSystemBackendTLS(policy) {
+ continue
+ }
+ upstreamTLS := &dxgateUpstreamTLS{
+ Mode: "simple",
+ SNI: string(policy.Spec.Validation.Hostname),
+ }
+ for _, target := range policy.Spec.TargetRefs {
+ if !isBackendTLSPolicyServiceTarget(target) {
+ continue
+ }
+ key := namespacedServiceKey(policy.Namespace,
string(target.Name))
+ if _, found := out[key]; !found {
+ out[key] = upstreamTLS
+ }
+ }
+ }
+ return out
+}
+
+func supportsSystemBackendTLS(policy *gateway.BackendTLSPolicy) bool {
+ if policy.Spec.Validation.Hostname == "" {
+ return false
+ }
+ wellKnown := policy.Spec.Validation.WellKnownCACertificates
+ return wellKnown != nil && *wellKnown ==
gateway.WellKnownCACertificatesSystem
+}
+
+func isBackendTLSPolicyServiceTarget(target
gateway.LocalPolicyTargetReferenceWithSectionName) bool {
+ if target.SectionName != nil {
+ return false
+ }
+ group := strings.TrimSpace(string(target.Group))
+ kind := strings.TrimSpace(string(target.Kind))
+ return (group == "" || group == "core") && strings.EqualFold(kind,
"Service")
+}
+
type dxgateBackendCircuitBreaker struct {
CircuitBreaker *dxgateCircuitBreaker
Outlier *dxgateOutlierDetection
diff --git
a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go
b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go
index 1764c6e6..da3cef09 100644
--- a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go
+++ b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go
@@ -122,7 +122,7 @@ func TestBuildDxgateRuntimeConfigFromHTTPRoute(t
*testing.T) {
},
}
- raw, hash, err := buildDxgateRuntimeConfig(gw,
[]*gatewayv1.HTTPRoute{route}, nil, "cluster.local")
+ raw, hash, err := buildDxgateRuntimeConfig(gw,
[]*gatewayv1.HTTPRoute{route}, nil, nil, nil, "cluster.local")
if err != nil {
t.Fatal(err)
}
@@ -220,7 +220,7 @@ func
TestBuildDxgateRuntimeConfigAppliesCircuitBreakerPolicy(t *testing.T) {
},
}
- raw, _, err := buildDxgateRuntimeConfig(gw,
[]*gatewayv1.HTTPRoute{route}, policies, "cluster.local")
+ raw, _, err := buildDxgateRuntimeConfig(gw,
[]*gatewayv1.HTTPRoute{route}, nil, nil, policies, "cluster.local")
if err != nil {
t.Fatal(err)
}
@@ -278,7 +278,7 @@ func
TestBuildDxgateRuntimeConfigFiltersUnattachedHTTPRoutes(t *testing.T) {
},
}
- raw, _, err := buildDxgateRuntimeConfig(gw,
[]*gatewayv1.HTTPRoute{route}, nil, "cluster.local")
+ raw, _, err := buildDxgateRuntimeConfig(gw,
[]*gatewayv1.HTTPRoute{route}, nil, nil, nil, "cluster.local")
if err != nil {
t.Fatal(err)
}
@@ -291,6 +291,96 @@ func
TestBuildDxgateRuntimeConfigFiltersUnattachedHTTPRoutes(t *testing.T) {
}
}
+func TestBuildDxgateRuntimeConfigExternalNameBackendTLS(t *testing.T) {
+ backendPort := gatewayv1.PortNumber(443)
+ hostname := gatewayv1.Hostname("httpbin-egress.app.svc.cluster.local")
+ wellKnown := gatewayv1.WellKnownCACertificatesSystem
+ gw := gatewayv1.Gateway{
+ ObjectMeta: metav1.ObjectMeta{Name: "egress", Namespace: "app",
ResourceVersion: "10"},
+ Spec: gatewayv1.GatewaySpec{
+ GatewayClassName: "dubbo",
+ Listeners: []gatewayv1.Listener{
+ {Name: "http", Protocol:
gatewayv1.HTTPProtocolType, Port: 80},
+ },
+ },
+ }
+ route := &gatewayv1.HTTPRoute{
+ ObjectMeta: metav1.ObjectMeta{Name: "httpbin", Namespace:
"app", ResourceVersion: "20"},
+ Spec: gatewayv1.HTTPRouteSpec{
+ CommonRouteSpec: gatewayv1.CommonRouteSpec{
+ ParentRefs: []gatewayv1.ParentReference{{Name:
"egress"}},
+ },
+ Hostnames: []gatewayv1.Hostname{hostname},
+ Rules: []gatewayv1.HTTPRouteRule{
+ {
+ BackendRefs: []gatewayv1.HTTPBackendRef{
+ {
+ BackendRef:
gatewayv1.BackendRef{
+
BackendObjectReference: gatewayv1.BackendObjectReference{
+ Name:
"httpbin-egress",
+ Port:
&backendPort,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+ service := &corev1.Service{
+ ObjectMeta: metav1.ObjectMeta{Name: "httpbin-egress",
Namespace: "app"},
+ Spec: corev1.ServiceSpec{
+ Type: corev1.ServiceTypeExternalName,
+ ExternalName: "httpbin.org",
+ Ports: []corev1.ServicePort{{Name: "https",
Port: 443}},
+ },
+ }
+ policy := &gatewayv1.BackendTLSPolicy{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "httpbin-tls",
+ Namespace: "app",
+ CreationTimestamp: metav1.NewTime(time.Unix(1, 0)),
+ },
+ Spec: gatewayv1.BackendTLSPolicySpec{
+ TargetRefs:
[]gatewayv1.LocalPolicyTargetReferenceWithSectionName{
+ {
+ LocalPolicyTargetReference:
gatewayv1.LocalPolicyTargetReference{
+ Group: gatewayv1.Group(""),
+ Kind:
gatewayv1.Kind("Service"),
+ Name:
gatewayv1.ObjectName("httpbin-egress"),
+ },
+ },
+ },
+ Validation: gatewayv1.BackendTLSPolicyValidation{
+ WellKnownCACertificates: &wellKnown,
+ Hostname:
gatewayv1.PreciseHostname("httpbin.org"),
+ },
+ },
+ }
+
+ raw, _, err := buildDxgateRuntimeConfig(gw,
[]*gatewayv1.HTTPRoute{route}, []*corev1.Service{service},
[]*gatewayv1.BackendTLSPolicy{policy}, nil, "cluster.local")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var cfg dxgateRuntimeConfig
+ if err := yaml.Unmarshal([]byte(raw), &cfg); err != nil {
+ t.Fatal(err)
+ }
+ if len(cfg.Clusters) != 1 {
+ t.Fatalf("clusters = %d, want 1", len(cfg.Clusters))
+ }
+ cluster := cfg.Clusters[0]
+ if got := cluster.Endpoints[0].Address; got != "httpbin.org" {
+ t.Fatalf("endpoint address = %q, want external DNS name", got)
+ }
+ if cluster.TLS == nil {
+ t.Fatal("cluster tls is nil")
+ }
+ if cluster.TLS.Mode != "simple" || cluster.TLS.SNI != "httpbin.org" {
+ t.Fatalf("unexpected tls config: %#v", cluster.TLS)
+ }
+}
+
func TestBuildDxgateBootstrapConfig(t *testing.T) {
raw, hash, err := buildDxgateBootstrapConfig(
"http://dubbod.dubbo-system.svc:26010";,
@@ -441,7 +531,7 @@ func TestKubeGatewayTemplateRendersDxgateResources(t
*testing.T) {
return inject.Config{Templates: templates}
},
}
- rendered, err := controller.render("gateway", TemplateInput{
+ input := TemplateInput{
Gateway: &gatewayv1.Gateway{
ObjectMeta: metav1.ObjectMeta{Name: "public",
Namespace: "app"},
},
@@ -456,7 +546,8 @@ func TestKubeGatewayTemplateRendersDxgateResources(t
*testing.T) {
SystemNamespace: "dubbo-system",
ClusterID: "Kubernetes",
DomainSuffix: "cluster.local",
- })
+ }
+ rendered, err := controller.render("gateway", input)
if err != nil {
t.Fatal(err)
}
@@ -487,6 +578,14 @@ func TestKubeGatewayTemplateRendersDxgateResources(t
*testing.T) {
if !strings.Contains(rendered[2], "inject.dubbo.apache.org/templates:
grpc-engine") {
t.Fatalf("deployment pod template did not request grpc-engine
injection:\n%s", rendered[2])
}
+ if !strings.Contains(rendered[2], `prometheus.io/scrape: "true"`) ||
+ !strings.Contains(rendered[2], "prometheus.io/path: /metrics")
||
+ !strings.Contains(rendered[2], `prometheus.io/port: "26021"`) {
+ t.Fatalf("deployment pod template did not render prometheus
scrape annotations:\n%s", rendered[2])
+ }
+ if strings.Contains(rendered[2], "DXGATE_OTEL_ENDPOINT") {
+ t.Fatalf("deployment rendered OTEL endpoint without
annotation:\n%s", rendered[2])
+ }
if !strings.Contains(rendered[2], "app.kubernetes.io/instance:
public-dubbo") {
t.Fatalf("deployment did not render stable dxgate instance
label:\n%s", rendered[2])
}
@@ -499,4 +598,14 @@ func TestKubeGatewayTemplateRendersDxgateResources(t
*testing.T) {
if !strings.Contains(rendered[3], "app.kubernetes.io/instance:
public-dubbo") {
t.Fatalf("service did not render stable dxgate instance
selector:\n%s", rendered[3])
}
+
+ input.OtelEndpoint = "http://tracing.dubbo-system.svc:4317";
+ rendered, err = controller.render("gateway", input)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(rendered[2], "DXGATE_OTEL_ENDPOINT") ||
+ !strings.Contains(rendered[2], `value:
"http://tracing.dubbo-system.svc:4317"`) {
+ t.Fatalf("deployment did not render dxgate OTEL endpoint:\n%s",
rendered[2])
+ }
}
diff --git a/dubbod/discovery/pkg/model/push_context.go
b/dubbod/discovery/pkg/model/push_context.go
index 6e82a0f3..b7c8a131 100644
--- a/dubbod/discovery/pkg/model/push_context.go
+++ b/dubbod/discovery/pkg/model/push_context.go
@@ -75,6 +75,7 @@ type PushContext struct {
ServiceIndex serviceIndex
virtualServiceIndex virtualServiceIndex
httpRouteIndex httpRouteIndex
+ backendTLSPolicyIndex backendTLSPolicyIndex
destinationRuleIndex destinationRuleIndex
serviceAccounts map[serviceAccountKey][]string
AuthenticationPolicies *AuthenticationPolicies
@@ -155,6 +156,14 @@ type httpRouteIndex struct {
hostToRoutes map[host.Name][]config.Config
}
+type backendTLSPolicyIndex struct {
+ serviceTLS map[string]BackendTLSSettings
+}
+
+type BackendTLSSettings struct {
+ SNI string
+}
+
type destinationRuleIndex struct {
namespaceLocal map[string]*consolidatedSubRules
exportedByNamespace map[string]*consolidatedSubRules
@@ -168,11 +177,12 @@ type consolidatedSubRules struct {
func NewPushContext() *PushContext {
return &PushContext{
- ServiceIndex: newServiceIndex(),
- virtualServiceIndex: newVirtualServiceIndex(),
- destinationRuleIndex: newDestinationRuleIndex(),
- serviceAccounts: map[serviceAccountKey][]string{},
- ProxyStatus: map[string]map[string]ProxyPushStatus{},
+ ServiceIndex: newServiceIndex(),
+ virtualServiceIndex: newVirtualServiceIndex(),
+ backendTLSPolicyIndex: backendTLSPolicyIndex{serviceTLS:
map[string]BackendTLSSettings{}},
+ destinationRuleIndex: newDestinationRuleIndex(),
+ serviceAccounts: map[serviceAccountKey][]string{},
+ ProxyStatus: map[string]map[string]ProxyPushStatus{},
}
}
@@ -589,6 +599,7 @@ func (ps *PushContext) createNewContext(env *Environment) {
// Initialize Kubernetes Gateway API resources if the controller is
enabled.
ps.initKubernetesGateways(env)
ps.initHTTPRoutes(env)
+ ps.initBackendTLSPolicies(env)
ps.initAuthenticationPolicies(env)
}
@@ -644,6 +655,15 @@ func (ps *PushContext) updateContext(env *Environment,
oldPushContext *PushConte
ps.httpRouteIndex = oldPushContext.httpRouteIndex
}
+ backendTLSPoliciesChanged := pushReq != nil &&
HasConfigsOfKind(pushReq.ConfigsUpdated, kind.BackendTLSPolicy)
+ if backendTLSPoliciesChanged {
+ log.Debugf("BackendTLSPolicies changed, re-initializing
BackendTLSPolicy index")
+ ps.initBackendTLSPolicies(env)
+ } else {
+ log.Debugf("BackendTLSPolicies unchanged, reusing old
BackendTLSPolicy index")
+ ps.backendTLSPolicyIndex = oldPushContext.backendTLSPolicyIndex
+ }
+
authnPoliciesChanged := pushReq != nil && (pushReq.Full ||
authPolicyKindsChanged(pushReq.ConfigsUpdated))
if authnPoliciesChanged || oldPushContext == nil ||
oldPushContext.AuthenticationPolicies == nil {
log.Debugf("security authentication policy changed (full=%v,
configsUpdatedContainingSecurityPolicy=%v), rebuilding authentication policies",
@@ -856,6 +876,62 @@ func (ps *PushContext) HTTPRouteForHost(hostname
host.Name) []config.Config {
return routes
}
+func (ps *PushContext) initBackendTLSPolicies(env *Environment) {
+ policies := sortConfigByCreationTime(env.List(gvk.BackendTLSPolicy,
NamespaceAll))
+ serviceTLS := map[string]BackendTLSSettings{}
+ for _, cfg := range policies {
+ spec, ok :=
cfg.Spec.(*sigsk8siogatewayapiapisv1.BackendTLSPolicySpec)
+ if !ok {
+ log.Debugf("BackendTLSPolicy %s/%s spec is not
BackendTLSPolicySpec", cfg.Namespace, cfg.Name)
+ continue
+ }
+ if !supportsSystemBackendTLS(spec) {
+ continue
+ }
+ settings := BackendTLSSettings{SNI:
string(spec.Validation.Hostname)}
+ for _, target := range spec.TargetRefs {
+ if !isBackendTLSPolicyServiceTarget(target) {
+ continue
+ }
+ key := backendTLSPolicyServiceKey(cfg.Namespace,
string(target.Name))
+ if _, found := serviceTLS[key]; !found {
+ serviceTLS[key] = settings
+ }
+ }
+ }
+ ps.backendTLSPolicyIndex.serviceTLS = serviceTLS
+ log.Debugf("indexed BackendTLSPolicies for %d services",
len(serviceTLS))
+}
+
+func (ps *PushContext) BackendTLSForService(namespace, name string)
(BackendTLSSettings, bool) {
+ if ps == nil || ps.backendTLSPolicyIndex.serviceTLS == nil {
+ return BackendTLSSettings{}, false
+ }
+ settings, found :=
ps.backendTLSPolicyIndex.serviceTLS[backendTLSPolicyServiceKey(namespace, name)]
+ return settings, found
+}
+
+func supportsSystemBackendTLS(spec
*sigsk8siogatewayapiapisv1.BackendTLSPolicySpec) bool {
+ if spec == nil || spec.Validation.Hostname == "" {
+ return false
+ }
+ wellKnown := spec.Validation.WellKnownCACertificates
+ return wellKnown != nil && *wellKnown ==
sigsk8siogatewayapiapisv1.WellKnownCACertificatesSystem
+}
+
+func isBackendTLSPolicyServiceTarget(target
sigsk8siogatewayapiapisv1.LocalPolicyTargetReferenceWithSectionName) bool {
+ if target.SectionName != nil {
+ return false
+ }
+ group := strings.TrimSpace(string(target.Group))
+ kind := strings.TrimSpace(string(target.Kind))
+ return (group == "" || group == "core") && strings.EqualFold(kind,
"Service")
+}
+
+func backendTLSPolicyServiceKey(namespace, name string) string {
+ return namespace + "/" + name
+}
+
// sortConfigBySelectorAndCreationTime sorts the list of config objects based
on creation time.
func sortConfigBySelectorAndCreationTime(configs []config.Config)
[]config.Config {
sort.Slice(configs, func(i, j int) bool {
diff --git a/dubbod/discovery/pkg/networking/grpcgen/cds.go
b/dubbod/discovery/pkg/networking/grpcgen/cds.go
index 320a8f04..001ed783 100644
--- a/dubbod/discovery/pkg/networking/grpcgen/cds.go
+++ b/dubbod/discovery/pkg/networking/grpcgen/cds.go
@@ -167,6 +167,12 @@ func (b *clusterBuilder) build() []*cluster.Cluster {
if newDefaultCluster != nil {
defaultCluster = newDefaultCluster
}
+ if b.node != nil && b.node.IsRouter() {
+ b.applyBackendTLSPolicy(defaultCluster)
+ for _, subsetCluster := range subsetClusters {
+ b.applyBackendTLSPolicy(subsetCluster)
+ }
+ }
out := make([]*cluster.Cluster, 0, 1+len(subsetClusters))
if defaultCluster != nil {
out = append(out, defaultCluster)
@@ -391,6 +397,28 @@ func (b *clusterBuilder) applyTLSForCluster(c
*cluster.Cluster, subset *networki
log.Debugf("applied %v TLS transport socket to cluster %s (SNI=%s)",
mode, c.Name, sni)
}
+func (b *clusterBuilder) applyBackendTLSPolicy(c *cluster.Cluster) {
+ if c == nil || c.TransportSocket != nil || b.svc == nil || b.push ==
nil {
+ return
+ }
+ if b.svc.Resolution != model.Alias ||
b.svc.Attributes.K8sAttributes.ExternalName == "" {
+ return
+ }
+ settings, found :=
b.push.BackendTLSForService(b.svc.Attributes.Namespace, b.svc.Attributes.Name)
+ if !found || settings.SNI == "" {
+ return
+ }
+ tlsContext := &tlsv1.UpstreamTlsContext{
+ CommonTlsContext: &tlsv1.CommonTlsContext{},
+ Sni: settings.SNI,
+ }
+ c.TransportSocket = &core.TransportSocket{
+ Name: "transport_sockets.tls",
+ ConfigType: &core.TransportSocket_TypedConfig{TypedConfig:
protoconv.MessageToAny(tlsContext)},
+ }
+ log.Debugf("applied BackendTLSPolicy simple TLS transport socket to
cluster %s (SNI=%s)", c.Name, settings.SNI)
+}
+
// buildUpstreamTLSContext builds an UpstreamTlsContext that conforms to gRPC
xDS expectations,
// reusing the common certificate-provider setup from buildCommonTLSContext.
func (b *clusterBuilder) buildUpstreamTLSContext(c *cluster.Cluster,
tlsSettings *networking.ClientTLSSettings) *tlsv1.UpstreamTlsContext {
diff --git a/dubbod/discovery/pkg/networking/grpcgen/cds_test.go
b/dubbod/discovery/pkg/networking/grpcgen/cds_test.go
new file mode 100644
index 00000000..996b5398
--- /dev/null
+++ b/dubbod/discovery/pkg/networking/grpcgen/cds_test.go
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package grpcgen
+
+import (
+ "testing"
+ "time"
+
+ "github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/model"
+ "github.com/apache/dubbo-kubernetes/pkg/config"
+ "github.com/apache/dubbo-kubernetes/pkg/config/schema/gvk"
+ cluster "github.com/kdubbo/xds-api/cluster/v1"
+ tlsv1 "github.com/kdubbo/xds-api/extensions/transport_sockets/tls/v1"
+ gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+)
+
+func TestBuildClustersExternalNameBackendTLSPolicyUsesSimpleTLS(t *testing.T) {
+ hostName := "httpbin-egress.app.svc.cluster.local"
+ service := newRDSTestService("httpbin-egress", "app", hostName, 443)
+ service.Resolution = model.Alias
+ service.Attributes.K8sAttributes.ExternalName = "httpbin.org"
+ push := newRDSTestPushContext(t, []config.Config{
+ newBackendTLSPolicyConfig("httpbin-tls", "app",
"httpbin-egress", "httpbin.org"),
+ }, []*model.Service{service})
+
+ resources := (&GrpcConfigGenerator{}).BuildClusters(&model.Proxy{
+ ID: "router~10.0.0.1~dxgate.app~app.svc.cluster.local",
+ Type: model.Router,
+ }, push, []string{"outbound|443||" + hostName})
+
+ if len(resources) != 1 {
+ t.Fatalf("resources = %d, want 1", len(resources))
+ }
+ c := &cluster.Cluster{}
+ if err := resources[0].GetResource().UnmarshalTo(c); err != nil {
+ t.Fatalf("unmarshal cluster: %v", err)
+ }
+ if c.GetTransportSocket() == nil {
+ t.Fatal("transport socket is nil")
+ }
+ tlsContext := &tlsv1.UpstreamTlsContext{}
+ if err :=
c.GetTransportSocket().GetTypedConfig().UnmarshalTo(tlsContext); err != nil {
+ t.Fatalf("unmarshal upstream tls context: %v", err)
+ }
+ if tlsContext.GetSni() != "httpbin.org" {
+ t.Fatalf("SNI = %q, want httpbin.org", tlsContext.GetSni())
+ }
+ if
tlsContext.GetCommonTlsContext().GetTlsCertificateCertificateProviderInstance()
!= nil {
+ t.Fatal("simple TLS should not configure a workload certificate
provider")
+ }
+}
+
+func newBackendTLSPolicyConfig(name, namespace, serviceName, hostname string)
config.Config {
+ wellKnown := gatewayv1.WellKnownCACertificatesSystem
+ return config.Config{
+ Meta: config.Meta{
+ GroupVersionKind: gvk.BackendTLSPolicy,
+ Name: name,
+ Namespace: namespace,
+ CreationTimestamp: time.Unix(1, 0),
+ },
+ Spec: &gatewayv1.BackendTLSPolicySpec{
+ TargetRefs:
[]gatewayv1.LocalPolicyTargetReferenceWithSectionName{
+ {
+ LocalPolicyTargetReference:
gatewayv1.LocalPolicyTargetReference{
+ Group: gatewayv1.Group(""),
+ Kind:
gatewayv1.Kind("Service"),
+ Name:
gatewayv1.ObjectName(serviceName),
+ },
+ },
+ },
+ Validation: gatewayv1.BackendTLSPolicyValidation{
+ WellKnownCACertificates: &wellKnown,
+ Hostname:
gatewayv1.PreciseHostname(hostname),
+ },
+ },
+ }
+}
diff --git a/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go
b/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go
index a6d78d2c..6ba24d46 100644
--- a/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go
+++ b/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go
@@ -96,6 +96,9 @@ func (b *EndpointBuilder)
BuildClusterLoadAssignmentWithGateways(endpointIndex *
if svcPort == nil {
return buildEmptyClusterLoadAssignment(b.clusterName)
}
+ if externalName := b.service.Attributes.K8sAttributes.ExternalName;
externalName != "" {
+ return buildDNSClusterLoadAssignment(b.clusterName,
externalName, uint32(svcPort.Port))
+ }
shards, ok := endpointIndex.ShardsForService(string(b.hostname),
b.service.Attributes.Namespace)
if !ok {
@@ -367,6 +370,33 @@ func buildEmptyClusterLoadAssignment(clusterName string)
*endpoint.ClusterLoadAs
}
}
+func buildDNSClusterLoadAssignment(clusterName, address string, port uint32)
*endpoint.ClusterLoadAssignment {
+ return &endpoint.ClusterLoadAssignment{
+ ClusterName: clusterName,
+ Endpoints: []*endpoint.LocalityLbEndpoints{
+ {
+ Locality: &core.Locality{},
+ LbEndpoints: []*endpoint.LbEndpoint{
+ {
+ HostIdentifier:
&endpoint.LbEndpoint_Endpoint{
+ Endpoint:
&endpoint.Endpoint{
+ Address:
util.BuildAddress(address, port),
+ },
+ },
+ HealthStatus:
core.HealthStatus_HEALTHY,
+ LoadBalancingWeight:
&wrapperspb.UInt32Value{
+ Value: 1,
+ },
+ },
+ },
+ LoadBalancingWeight: &wrapperspb.UInt32Value{
+ Value: 1,
+ },
+ },
+ },
+ }
+}
+
// Cacheable implements model.XdsCacheEntry
func (b *EndpointBuilder) Cacheable() bool {
return b.service != nil
diff --git a/dubbod/discovery/pkg/xds/endpoints/endpoint_builder_test.go
b/dubbod/discovery/pkg/xds/endpoints/endpoint_builder_test.go
index 40677091..fff6528c 100644
--- a/dubbod/discovery/pkg/xds/endpoints/endpoint_builder_test.go
+++ b/dubbod/discovery/pkg/xds/endpoints/endpoint_builder_test.go
@@ -38,6 +38,26 @@ func
TestBuildClusterLoadAssignmentKeepsAppPortWithoutDUBBOMutual(t *testing.T)
}
}
+func TestBuildClusterLoadAssignmentUsesExternalNameDNS(t *testing.T) {
+ hostname := host.Name("httpbin-egress.app.svc.cluster.local")
+ svc := newEndpointTestService("httpbin-egress", "app",
string(hostname), 443)
+ svc.Resolution = model.Alias
+ svc.Attributes.K8sAttributes.ExternalName = "httpbin.org"
+ push := newEndpointTestPushContext(t, nil, []*model.Service{svc})
+ index := model.NewEndpointIndex(model.DisabledCache{})
+
+ clusterName := model.BuildSubsetKey(model.TrafficDirectionOutbound, "",
hostname, 443)
+ builder := NewEndpointBuilder(clusterName, newEndpointTestProxy(), push)
+ cla := builder.BuildClusterLoadAssignment(index)
+
+ if got := firstEndpointAddress(t, cla); got != "httpbin.org" {
+ t.Fatalf("endpoint address = %q, want external DNS name", got)
+ }
+ if got := firstEndpointPort(t, cla); got != 443 {
+ t.Fatalf("endpoint port = %d, want service port 443", got)
+ }
+}
+
func TestBuildClusterLoadAssignmentUsesEastWestGatewayForRemoteShard(t
*testing.T) {
hostname := host.Name("nginx.app.svc.cluster.local")
svc := newEndpointTestService("nginx", "app", string(hostname), 80)
diff --git a/manifests/charts/dubbod/files/kube-gateway.yaml
b/manifests/charts/dubbod/files/kube-gateway.yaml
index de2e38b8..d0962904 100644
--- a/manifests/charts/dubbod/files/kube-gateway.yaml
+++ b/manifests/charts/dubbod/files/kube-gateway.yaml
@@ -68,6 +68,9 @@ spec:
dubbo.apache.org/rev: {{ .Revision | quote }}
gateway.dubbo.apache.org/bootstrap-hash: {{ .BootstrapConfigHash |
quote }}
inject.dubbo.apache.org/templates: grpc-engine
+ prometheus.io/scrape: "true"
+ prometheus.io/path: /metrics
+ prometheus.io/port: "26021"
spec:
serviceAccountName: {{ .ServiceAccount }}
containers:
@@ -81,6 +84,10 @@ spec:
value: 0.0.0.0:8080
- name: DXGATE_ADMIN_ADDR
value: 0.0.0.0:26021
+{{- if .OtelEndpoint }}
+ - name: DXGATE_OTEL_ENDPOINT
+ value: {{ .OtelEndpoint | quote }}
+{{- end }}
- name: POD_NAME
valueFrom:
fieldRef:
diff --git a/pkg/config/schema/collections/collections.gen.go
b/pkg/config/schema/collections/collections.gen.go
index 60ace61c..c454cc1a 100755
--- a/pkg/config/schema/collections/collections.gen.go
+++ b/pkg/config/schema/collections/collections.gen.go
@@ -42,6 +42,24 @@ var (
ValidateProto: validation.EmptyValidate,
}.MustBuild()
+ BackendTLSPolicy = resource.Builder{
+ Identifier: "BackendTLSPolicy",
+ Group: "gateway.networking.k8s.io",
+ Kind: "BackendTLSPolicy",
+ Plural: "backendtlspolicies",
+ Version: "v1",
+ VersionAliases: []string{
+ "v1",
+ },
+ Proto: "k8s.io.gateway_api.api.v1alpha1.BackendTLSPolicySpec",
StatusProto: "k8s.io.gateway_api.api.v1alpha1.PolicyStatus",
+ ReflectType:
reflect.TypeOf(&sigsk8siogatewayapiapisv1.BackendTLSPolicySpec{}).Elem(),
StatusType: reflect.TypeOf(&sigsk8siogatewayapiapisv1.PolicyStatus{}).Elem(),
+ ProtoPackage: "sigs.k8s.io/gateway-api/apis/v1", StatusPackage:
"sigs.k8s.io/gateway-api/apis/v1",
+ ClusterScoped: false,
+ Synthetic: false,
+ Builtin: false,
+ ValidateProto: validation.EmptyValidate,
+ }.MustBuild()
+
CircuitBreakerPolicy = resource.Builder{
Identifier: "CircuitBreakerPolicy",
Group: "networking.dubbo.apache.org",
@@ -429,6 +447,7 @@ var (
// All contains all collections in the system.
All = collection.NewSchemasBuilder().
MustAdd(AuthorizationPolicy).
+ MustAdd(BackendTLSPolicy).
MustAdd(CircuitBreakerPolicy).
MustAdd(ConfigMap).
MustAdd(CustomResourceDefinition).
@@ -458,6 +477,7 @@ var (
// Kube contains only kubernetes collections.
Kube = collection.NewSchemasBuilder().
+ MustAdd(BackendTLSPolicy).
MustAdd(ConfigMap).
MustAdd(CustomResourceDefinition).
MustAdd(DaemonSet).
@@ -492,6 +512,7 @@ var (
// dubboGatewayAPI contains only collections used by Dubbo, including
the full Gateway API.
dubboGatewayAPI = collection.NewSchemasBuilder().
MustAdd(AuthorizationPolicy).
+ MustAdd(BackendTLSPolicy).
MustAdd(CircuitBreakerPolicy).
MustAdd(GatewayClass).
MustAdd(HTTPRoute).
@@ -503,6 +524,7 @@ var (
// dubboStableGatewayAPI contains only collections used by Dubbo,
including beta+ Gateway API.
dubboStableGatewayAPI = collection.NewSchemasBuilder().
MustAdd(AuthorizationPolicy).
+ MustAdd(BackendTLSPolicy).
MustAdd(CircuitBreakerPolicy).
MustAdd(GatewayClass).
MustAdd(HTTPRoute).
diff --git a/pkg/config/schema/gvk/resources.gen.go
b/pkg/config/schema/gvk/resources.gen.go
index f1540b62..5e3768ba 100755
--- a/pkg/config/schema/gvk/resources.gen.go
+++ b/pkg/config/schema/gvk/resources.gen.go
@@ -11,6 +11,8 @@ import (
var (
AuthorizationPolicy = config.GroupVersionKind{Group:
"security.dubbo.apache.org", Version: "v1alpha3", Kind: "AuthorizationPolicy"}
+ BackendTLSPolicy = config.GroupVersionKind{Group:
"gateway.networking.k8s.io", Version: "v1", Kind: "BackendTLSPolicy"}
+ BackendTLSPolicy_v1 = config.GroupVersionKind{Group:
"gateway.networking.k8s.io", Version: "v1", Kind: "BackendTLSPolicy"}
CircuitBreakerPolicy = config.GroupVersionKind{Group:
"networking.dubbo.apache.org", Version: "v1alpha3", Kind:
"CircuitBreakerPolicy"}
ConfigMap = config.GroupVersionKind{Group: "",
Version: "v1", Kind: "ConfigMap"}
CustomResourceDefinition = config.GroupVersionKind{Group:
"apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"}
@@ -46,6 +48,10 @@ func ToGVR(g config.GroupVersionKind)
(schema.GroupVersionResource, bool) {
switch g {
case AuthorizationPolicy:
return gvr.AuthorizationPolicy, true
+ case BackendTLSPolicy:
+ return gvr.BackendTLSPolicy, true
+ case BackendTLSPolicy_v1:
+ return gvr.BackendTLSPolicy_v1, true
case CircuitBreakerPolicy:
return gvr.CircuitBreakerPolicy, true
case ConfigMap:
@@ -111,6 +117,8 @@ func MustToKind(g config.GroupVersionKind) kind.Kind {
switch g {
case AuthorizationPolicy:
return kind.AuthorizationPolicy
+ case BackendTLSPolicy:
+ return kind.BackendTLSPolicy
case CircuitBreakerPolicy:
return kind.CircuitBreakerPolicy
case ConfigMap:
@@ -181,6 +189,8 @@ func FromGVR(g schema.GroupVersionResource)
(config.GroupVersionKind, bool) {
switch g {
case gvr.AuthorizationPolicy:
return AuthorizationPolicy, true
+ case gvr.BackendTLSPolicy:
+ return BackendTLSPolicy, true
case gvr.CircuitBreakerPolicy:
return CircuitBreakerPolicy, true
case gvr.ConfigMap:
diff --git a/pkg/config/schema/gvr/resources.gen.go
b/pkg/config/schema/gvr/resources.gen.go
index 52be18a4..21955199 100755
--- a/pkg/config/schema/gvr/resources.gen.go
+++ b/pkg/config/schema/gvr/resources.gen.go
@@ -6,6 +6,8 @@ import "k8s.io/apimachinery/pkg/runtime/schema"
var (
AuthorizationPolicy = schema.GroupVersionResource{Group:
"security.dubbo.apache.org", Version: "v1alpha3", Resource:
"authorizationpolicies"}
+ BackendTLSPolicy = schema.GroupVersionResource{Group:
"gateway.networking.k8s.io", Version: "v1", Resource: "backendtlspolicies"}
+ BackendTLSPolicy_v1 = schema.GroupVersionResource{Group:
"gateway.networking.k8s.io", Version: "v1", Resource: "backendtlspolicies"}
CircuitBreakerPolicy = schema.GroupVersionResource{Group:
"networking.dubbo.apache.org", Version: "v1alpha3", Resource:
"circuitbreakerpolicies"}
ConfigMap = schema.GroupVersionResource{Group: "",
Version: "v1", Resource: "configmaps"}
CustomResourceDefinition = schema.GroupVersionResource{Group:
"apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions"}
@@ -40,6 +42,10 @@ func IsClusterScoped(g schema.GroupVersionResource) bool {
switch g {
case AuthorizationPolicy:
return false
+ case BackendTLSPolicy:
+ return false
+ case BackendTLSPolicy_v1:
+ return false
case CircuitBreakerPolicy:
return false
case ConfigMap:
diff --git a/pkg/config/schema/kind/resources.gen.go
b/pkg/config/schema/kind/resources.gen.go
index f31e2742..40ab69ab 100755
--- a/pkg/config/schema/kind/resources.gen.go
+++ b/pkg/config/schema/kind/resources.gen.go
@@ -6,6 +6,7 @@ const (
Unknown Kind = iota
Address
AuthorizationPolicy
+ BackendTLSPolicy
CircuitBreakerPolicy
ConfigMap
CustomResourceDefinition
@@ -40,6 +41,8 @@ func (k Kind) String() string {
return "Address"
case AuthorizationPolicy:
return "AuthorizationPolicy"
+ case BackendTLSPolicy:
+ return "BackendTLSPolicy"
case CircuitBreakerPolicy:
return "CircuitBreakerPolicy"
case ConfigMap:
@@ -103,6 +106,8 @@ func FromString(s string) Kind {
return Address
case "AuthorizationPolicy":
return AuthorizationPolicy
+ case "BackendTLSPolicy":
+ return BackendTLSPolicy
case "CircuitBreakerPolicy":
return CircuitBreakerPolicy
case "ConfigMap":
diff --git a/pkg/config/schema/kubeclient/resources.gen.go
b/pkg/config/schema/kubeclient/resources.gen.go
index e07946d8..c571b09b 100755
--- a/pkg/config/schema/kubeclient/resources.gen.go
+++ b/pkg/config/schema/kubeclient/resources.gen.go
@@ -34,6 +34,8 @@ func GetWriteClient[T runtime.Object](c ClientGetter,
namespace string) ktypes.W
switch any(ptr.Empty[T]()).(type) {
case
*apigithubcomapachedubbokubernetesapisecurityv1alpha3.AuthorizationPolicy:
return
c.Dubbo().SecurityV1alpha3().AuthorizationPolicies(namespace).(ktypes.WriteAPI[T])
+ case *sigsk8siogatewayapiapisv1.BackendTLSPolicy:
+ return
c.GatewayAPI().GatewayV1().BackendTLSPolicies(namespace).(ktypes.WriteAPI[T])
case
*apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy:
return
c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(namespace).(ktypes.WriteAPI[T])
case *k8sioapicorev1.ConfigMap:
@@ -91,6 +93,8 @@ func GetClient[T, TL runtime.Object](c ClientGetter,
namespace string) ktypes.Re
switch any(ptr.Empty[T]()).(type) {
case
*apigithubcomapachedubbokubernetesapisecurityv1alpha3.AuthorizationPolicy:
return
c.Dubbo().SecurityV1alpha3().AuthorizationPolicies(namespace).(ktypes.ReadWriteAPI[T,
TL])
+ case *sigsk8siogatewayapiapisv1.BackendTLSPolicy:
+ return
c.GatewayAPI().GatewayV1().BackendTLSPolicies(namespace).(ktypes.ReadWriteAPI[T,
TL])
case
*apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy:
return
c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(namespace).(ktypes.ReadWriteAPI[T,
TL])
case *k8sioapicorev1.ConfigMap:
@@ -148,6 +152,8 @@ func gvrToObject(g schema.GroupVersionResource)
runtime.Object {
switch g {
case gvr.AuthorizationPolicy:
return
&apigithubcomapachedubbokubernetesapisecurityv1alpha3.AuthorizationPolicy{}
+ case gvr.BackendTLSPolicy:
+ return &sigsk8siogatewayapiapisv1.BackendTLSPolicy{}
case gvr.CircuitBreakerPolicy:
return
&apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{}
case gvr.ConfigMap:
@@ -213,6 +219,13 @@ func getInformerFiltered(c ClientGetter, opts
ktypes.InformerOptions, g schema.G
w = func(options metav1.ListOptions) (watch.Interface, error) {
return
c.Dubbo().SecurityV1alpha3().AuthorizationPolicies(opts.Namespace).Watch(context.Background(),
options)
}
+ case gvr.BackendTLSPolicy:
+ l = func(options metav1.ListOptions) (runtime.Object, error) {
+ return
c.GatewayAPI().GatewayV1().BackendTLSPolicies(opts.Namespace).List(context.Background(),
options)
+ }
+ w = func(options metav1.ListOptions) (watch.Interface, error) {
+ return
c.GatewayAPI().GatewayV1().BackendTLSPolicies(opts.Namespace).Watch(context.Background(),
options)
+ }
case gvr.CircuitBreakerPolicy:
l = func(options metav1.ListOptions) (runtime.Object, error) {
return
c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(opts.Namespace).List(context.Background(),
options)
diff --git a/pkg/config/schema/kubetypes/resources.gen.go
b/pkg/config/schema/kubetypes/resources.gen.go
index 910b664f..38722483 100755
--- a/pkg/config/schema/kubetypes/resources.gen.go
+++ b/pkg/config/schema/kubetypes/resources.gen.go
@@ -27,6 +27,8 @@ func getGvk(obj any) (config.GroupVersionKind, bool) {
return gvk.AuthorizationPolicy, true
case
*apigithubcomapachedubbokubernetesapisecurityv1alpha3.AuthorizationPolicy:
return gvk.AuthorizationPolicy, true
+ case *sigsk8siogatewayapiapisv1.BackendTLSPolicy:
+ return gvk.BackendTLSPolicy, true
case *githubcomkdubboapinetworkingv1alpha3.CircuitBreakerPolicy:
return gvk.CircuitBreakerPolicy, true
case
*apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy:
diff --git a/pkg/config/schema/metadata.yaml b/pkg/config/schema/metadata.yaml
index c62921b0..5359af4a 100644
--- a/pkg/config/schema/metadata.yaml
+++ b/pkg/config/schema/metadata.yaml
@@ -191,6 +191,17 @@ resources:
statusProto: "k8s.io.gateway_api.api.v1alpha1.HTTPRouteStatus"
statusProtoPackage: "sigs.k8s.io/gateway-api/apis/v1"
+ - kind: "BackendTLSPolicy"
+ plural: "backendtlspolicies"
+ group: "gateway.networking.k8s.io"
+ version: "v1"
+ versionAliases:
+ - "v1"
+ protoPackage: "sigs.k8s.io/gateway-api/apis/v1"
+ proto: "k8s.io.gateway_api.api.v1alpha1.BackendTLSPolicySpec"
+ statusProto: "k8s.io.gateway_api.api.v1alpha1.PolicyStatus"
+ statusProtoPackage: "sigs.k8s.io/gateway-api/apis/v1"
+
## Dubbo resources
- kind: AuthorizationPolicy
plural: "authorizationpolicies"
diff --git a/samples/addons/grafana.yaml b/samples/addons/grafana.yaml
index 98651b8c..406eebd8 100644
--- a/samples/addons/grafana.yaml
+++ b/samples/addons/grafana.yaml
@@ -804,6 +804,163 @@ data:
],
"title": "XDS Reject Rate",
"type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 40
+ },
+ "id": 19,
+ "panels": [],
+ "title": "Gateway Traffic",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 41
+ },
+ "id": 20,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "sum by (route, cluster)
(rate(dxgate_http_route_requests_total[5m]))",
+ "legendFormat": "{{route}} / {{cluster}}",
+ "refId": "A"
+ }
+ ],
+ "title": "HTTP Request Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 41
+ },
+ "id": 21,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "sum by (route, cluster)
(rate(dxgate_http_route_failures_total[5m]))",
+ "legendFormat": "{{route}} / {{cluster}}",
+ "refId": "A"
+ }
+ ],
+ "title": "HTTP Failure Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 41
+ },
+ "id": 22,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.95, sum by (le, route, cluster)
(rate(dxgate_http_route_latency_ms_bucket[5m])))",
+ "legendFormat": "{{route}} / {{cluster}}",
+ "refId": "A"
+ }
+ ],
+ "title": "HTTP Upstream P95",
+ "type": "timeseries"
}
],
"refresh": "15s",
diff --git a/samples/addons/prometheus.yaml b/samples/addons/prometheus.yaml
index 7d46e93f..17f1d459 100644
--- a/samples/addons/prometheus.yaml
+++ b/samples/addons/prometheus.yaml
@@ -22,6 +22,34 @@ metadata:
app.kubernetes.io/name: prometheus
app.kubernetes.io/part-of: dubbo-observability
---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: dubbo-prometheus
+ labels:
+ app.kubernetes.io/name: prometheus
+ app.kubernetes.io/part-of: dubbo-observability
+rules:
+- apiGroups: [""]
+ resources: ["nodes", "pods", "services", "endpoints"]
+ verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: dubbo-prometheus
+ labels:
+ app.kubernetes.io/name: prometheus
+ app.kubernetes.io/part-of: dubbo-observability
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: dubbo-prometheus
+subjects:
+- kind: ServiceAccount
+ name: prometheus
+ namespace: dubbo-system
+---
apiVersion: v1
kind: ConfigMap
metadata:
@@ -50,6 +78,29 @@ data:
labels:
app: dubbod
namespace: dubbo-system
+
+ - job_name: kubernetes-pods
+ kubernetes_sd_configs:
+ - role: pod
+ relabel_configs:
+ - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+ action: keep
+ regex: "true"
+ - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+ action: replace
+ target_label: __metrics_path__
+ regex: (.+)
+ - source_labels: [__address__,
__meta_kubernetes_pod_annotation_prometheus_io_port]
+ action: replace
+ regex: ([^:]+)(?::\d+)?;(\d+)
+ replacement: $1:$2
+ target_label: __address__
+ - source_labels: [__meta_kubernetes_namespace]
+ target_label: namespace
+ - source_labels: [__meta_kubernetes_pod_name]
+ target_label: pod
+ - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+ target_label: app
---
apiVersion: v1
kind: Service
diff --git a/samples/addons/prometheus.yaml b/samples/addons/tracing.yaml
similarity index 50%
copy from samples/addons/prometheus.yaml
copy to samples/addons/tracing.yaml
index 7d46e93f..f63e7ced 100644
--- a/samples/addons/prometheus.yaml
+++ b/samples/addons/tracing.yaml
@@ -16,117 +16,91 @@
apiVersion: v1
kind: ServiceAccount
metadata:
- name: prometheus
+ name: tracing
namespace: dubbo-system
labels:
- app.kubernetes.io/name: prometheus
+ app.kubernetes.io/name: tracing
app.kubernetes.io/part-of: dubbo-observability
---
apiVersion: v1
-kind: ConfigMap
-metadata:
- name: prometheus
- namespace: dubbo-system
- labels:
- app.kubernetes.io/name: prometheus
- app.kubernetes.io/part-of: dubbo-observability
-data:
- prometheus.yml: |
- global:
- scrape_interval: 15s
- evaluation_interval: 15s
-
- scrape_configs:
- - job_name: prometheus
- static_configs:
- - targets:
- - localhost:9090
-
- - job_name: dubbod
- metrics_path: /metrics
- static_configs:
- - targets:
- - dubbod.dubbo-system.svc:8080
- labels:
- app: dubbod
- namespace: dubbo-system
----
-apiVersion: v1
kind: Service
metadata:
- name: prometheus
+ name: tracing
namespace: dubbo-system
labels:
- app.kubernetes.io/name: prometheus
+ app.kubernetes.io/name: tracing
app.kubernetes.io/part-of: dubbo-observability
spec:
+ selector:
+ app.kubernetes.io/name: tracing
ports:
- - name: http
- port: 9090
+ - name: http-query
+ port: 16686
+ targetPort: http-query
+ protocol: TCP
+ - name: grpc-otlp
+ port: 4317
+ targetPort: grpc-otlp
+ protocol: TCP
+ - name: http-otlp
+ port: 4318
+ targetPort: http-otlp
protocol: TCP
- targetPort: 9090
- selector:
- app.kubernetes.io/name: prometheus
---
apiVersion: apps/v1
kind: Deployment
metadata:
- name: prometheus
+ name: tracing
namespace: dubbo-system
labels:
- app.kubernetes.io/name: prometheus
+ app.kubernetes.io/name: tracing
app.kubernetes.io/part-of: dubbo-observability
spec:
replicas: 1
selector:
matchLabels:
- app.kubernetes.io/name: prometheus
+ app.kubernetes.io/name: tracing
template:
metadata:
labels:
- app.kubernetes.io/name: prometheus
+ app.kubernetes.io/name: tracing
app.kubernetes.io/part-of: dubbo-observability
spec:
- serviceAccountName: prometheus
+ serviceAccountName: tracing
containers:
- - name: prometheus
- image: prom/prometheus:v3.5.0
+ - name: jaeger
+ image: docker.io/jaegertracing/all-in-one:1.60
imagePullPolicy: IfNotPresent
- args:
- - --config.file=/etc/prometheus/prometheus.yml
- - --storage.tsdb.path=/prometheus
- - --web.enable-lifecycle
+ env:
+ - name: COLLECTOR_OTLP_ENABLED
+ value: "true"
+ - name: QUERY_BASE_PATH
+ value: /
ports:
- - name: http
- containerPort: 9090
+ - name: http-query
+ containerPort: 16686
+ protocol: TCP
+ - name: grpc-otlp
+ containerPort: 4317
+ protocol: TCP
+ - name: http-otlp
+ containerPort: 4318
protocol: TCP
readinessProbe:
httpGet:
- path: /-/ready
- port: 9090
+ path: /
+ port: http-query
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
- path: /-/healthy
- port: 9090
- initialDelaySeconds: 30
+ path: /
+ port: http-query
+ initialDelaySeconds: 20
periodSeconds: 15
resources:
requests:
cpu: 100m
- memory: 256Mi
+ memory: 128Mi
limits:
memory: 512Mi
- volumeMounts:
- - name: config
- mountPath: /etc/prometheus
- readOnly: true
- - name: storage
- mountPath: /prometheus
- volumes:
- - name: config
- configMap:
- name: prometheus
- - name: storage
- emptyDir: {}
