This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch ondemandlog2 in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit e5ab058597de33078d9c04f3150c788b72b4f813 Author: kezhenxu94 <[email protected]> AuthorDate: Wed Jun 1 15:29:53 2022 +0800 Allow querying logs only by instance to simplify frontend query --- .../analysis/manual/instance/InstanceTraffic.java | 2 + .../oap/query/graphql/GraphQLQueryProvider.java | 5 +- .../query/graphql/resolver/OndemandLogQuery.java | 94 +++++++++++++--------- ....java => OndemandContainergQueryCondition.java} | 11 +-- .../graphql/type/OndemandLogQueryCondition.java | 2 - .../src/main/resources/query-protocol | 2 +- .../main/resources/otel-oc-rules/k8s-instance.yaml | 23 ++++++ 7 files changed, 86 insertions(+), 53 deletions(-) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java index a36c29168d..c413495b81 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java @@ -175,6 +175,8 @@ public class InstanceTraffic extends Metrics { } public static class PropertyUtil { + public static final String NAMESPACE = "namespace"; + public static final String POD = "pod"; public static final String LANGUAGE = "language"; public static final String IPV4 = "ipv4"; public static final String IPV4S = "ipv4s"; diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java index 919fdbb038..f4f694a96e 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java @@ -78,6 +78,7 @@ public class GraphQLQueryProvider extends ModuleProvider { @Override public void prepare() throws ServiceNotProvidedException { + final MetadataQueryV2 metadataQueryV2 = new MetadataQueryV2(getManager()); schemaBuilder.file("query-protocol/common.graphqls") .resolvers(new Query(), new Mutation(), new HealthQuery(getManager())) .file("query-protocol/metadata.graphqls") @@ -118,14 +119,14 @@ public class GraphQLQueryProvider extends ModuleProvider { .file("query-protocol/event.graphqls") .resolvers(new EventQuery(getManager())) .file("query-protocol/metadata-v2.graphqls") - .resolvers(new MetadataQueryV2(getManager())) + .resolvers(metadataQueryV2) .file("query-protocol/ebpf-profiling.graphqls") .resolvers(new EBPFProcessProfilingQuery(getManager()), new EBPFProcessProfilingMutation(getManager())); if (config.isEnableOnDemandPodLog()) { schemaBuilder .file("query-protocol/ondemand-pod-log.graphqls") - .resolvers(new OndemandLogQuery()); + .resolvers(new OndemandLogQuery(metadataQueryV2)); } schemaBuilder.scalars(ExtendedScalars.GraphQLLong); diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java index 458572ba16..e768164a46 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.lang.reflect.Type; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import com.google.common.base.Predicate; import com.google.common.base.Splitter; @@ -34,64 +33,86 @@ import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import org.apache.skywalking.oap.query.graphql.type.InternalLog; import org.apache.skywalking.oap.query.graphql.type.LogAdapter; +import org.apache.skywalking.oap.query.graphql.type.OndemandContainergQueryCondition; import org.apache.skywalking.oap.query.graphql.type.OndemandLogQueryCondition; -import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic.PropertyUtil; import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.Attribute; import org.apache.skywalking.oap.server.core.query.type.Log; import org.apache.skywalking.oap.server.core.query.type.Logs; +import org.apache.skywalking.oap.server.core.query.type.ServiceInstance; import org.apache.skywalking.oap.server.library.util.StringUtil; import graphql.kickstart.tools.GraphQLQueryResolver; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.Configuration; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1Container; -import io.kubernetes.client.openapi.models.V1Namespace; -import io.kubernetes.client.openapi.models.V1NamespaceList; import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1PodSpec; import io.kubernetes.client.util.Config; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j +@RequiredArgsConstructor public class OndemandLogQuery implements GraphQLQueryResolver { private final Gson gson = new Gson(); private final Type responseType = new TypeToken<Map<String, Object>>() { }.getType(); private CoreV1Api kApi; - public List<String> listNamespaces() throws IOException { - try { - final V1NamespaceList nsList = - kApi().listNamespace(null, null, null, null, null, null, null, null, null, null); - return nsList - .getItems() - .stream() - .map(V1Namespace::getMetadata) - .filter(Objects::nonNull) - .map(V1ObjectMeta::getName) - .collect(Collectors.toList()); - } catch (ApiException e) { - log.error("Failed to list namespaces from Kubernetes, {}", e.getResponseBody(), e); + private final MetadataQueryV2 metadataQuery; - Map<String, Object> responseBody = gson.fromJson(e.getResponseBody(), responseType); - String message = responseBody.getOrDefault("message", e.getCode()).toString(); - throw new RuntimeException(message); - } + public List<String> listContainers(final OndemandContainergQueryCondition condition) + throws IOException { + final ServiceInstance instance = + metadataQuery.getInstance(condition.getServiceInstanceId()); + final Map<String, String> attributesMap = ensureInstanceHasPodProperties(instance); + final String ns = attributesMap.get(PropertyUtil.NAMESPACE); + final String pod = attributesMap.get(PropertyUtil.POD); + return listContainers(ns, pod); } - public List<String> listContainers(final OndemandLogQueryCondition condition) + public Logs ondemandPodLogs(final OndemandLogQueryCondition condition) throws IOException { - final String ns = condition.getNamespace(); - final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition = - IDManager.ServiceInstanceID.analysisId(condition.getServiceInstanceId()); - final String instanceName = instanceIDDefinition.getName(); + final ServiceInstance instance = + metadataQuery.getInstance(condition.getServiceInstanceId()); + final Map<String, String> attributesMap = ensureInstanceHasPodProperties(instance); + final String ns = attributesMap.get(PropertyUtil.NAMESPACE); + final String pod = attributesMap.get(PropertyUtil.POD); + return ondemandPodLogs(ns, pod, condition); + } + protected Map<String, String> ensureInstanceHasPodProperties(final ServiceInstance instance) { + if (instance == null) { + throw new IllegalArgumentException("No such instance"); + } + final List<Attribute> attributes = instance.getAttributes(); + if (attributes == null || attributes.isEmpty()) { + throw new IllegalArgumentException( + "Service instance has no valid properties to locate the Pod"); + } + final Map<String, String> attributesMap = + attributes + .stream() + .collect(Collectors.toMap(Attribute::getName, Attribute::getValue)); + if (!attributesMap.containsKey(PropertyUtil.NAMESPACE) + || !attributesMap.containsKey(PropertyUtil.POD)) { + throw new IllegalArgumentException( + "Service instance has no valid properties to locate the Pod"); + } + return attributesMap; + } + + protected List<String> listContainers( + final String namespace, + final String podName) throws IOException { try { - final V1Pod pod = kApi().readNamespacedPod(instanceName, ns, null); + final V1Pod pod = kApi().readNamespacedPod(podName, namespace, null); final V1PodSpec spec = pod.getSpec(); if (isNull(spec)) { - throw new RuntimeException(String.format("No spec: %s:%s", ns, instanceName)); + throw new RuntimeException(String.format("No spec: %s:%s", namespace, podName)); } final List<String> containers = spec.getContainers().stream() @@ -114,23 +135,20 @@ public class OndemandLogQuery implements GraphQLQueryResolver { } } - public Logs ondemandPodLogs(OndemandLogQueryCondition condition) - throws IOException { - final String ns = condition.getNamespace(); - final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition = - IDManager.ServiceInstanceID.analysisId(condition.getServiceInstanceId()); - final String instanceName = instanceIDDefinition.getName(); - + protected Logs ondemandPodLogs( + final String namespace, + final String podName, + final OndemandLogQueryCondition condition) throws IOException { try { - final V1Pod pod = kApi().readNamespacedPod(instanceName, ns, null); + final V1Pod pod = kApi().readNamespacedPod(podName, namespace, null); final V1ObjectMeta podMetadata = pod.getMetadata(); if (isNull(podMetadata)) { throw new RuntimeException( - String.format("No such instance: %s:%s", ns, instanceName)); + String.format("No such instance: %s:%s", namespace, podName)); } final V1PodSpec spec = pod.getSpec(); if (isNull(spec)) { - throw new RuntimeException(String.format("No spec: %s:%s", ns, instanceName)); + throw new RuntimeException(String.format("No spec: %s:%s", podName, podName)); } final Duration duration = new Duration(); diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandContainergQueryCondition.java similarity index 69% copy from oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java copy to oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandContainergQueryCondition.java index d186e1d496..754412e0e6 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandContainergQueryCondition.java @@ -19,20 +19,11 @@ package org.apache.skywalking.oap.query.graphql.type; -import java.util.Collections; -import java.util.List; -import org.apache.skywalking.oap.server.core.query.input.Duration; import lombok.Getter; import lombok.Setter; @Getter @Setter -public class OndemandLogQueryCondition { - private String namespace; - private String container; - private String serviceId; +public class OndemandContainergQueryCondition { private String serviceInstanceId; - private Duration duration; - private List<String> keywordsOfContent = Collections.emptyList(); - private List<String> excludingKeywordsOfContent = Collections.emptyList(); } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java index d186e1d496..43397b2748 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java @@ -28,9 +28,7 @@ import lombok.Setter; @Getter @Setter public class OndemandLogQueryCondition { - private String namespace; private String container; - private String serviceId; private String serviceInstanceId; private Duration duration; private List<String> keywordsOfContent = Collections.emptyList(); diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index edf5871a3a..faf3a76808 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit edf5871a3a2e8f7300eb06175240b56c21072fe9 +Subproject commit faf3a768087825258f64dd782a4dc405dc271cf8 diff --git a/oap-server/server-starter/src/main/resources/otel-oc-rules/k8s-instance.yaml b/oap-server/server-starter/src/main/resources/otel-oc-rules/k8s-instance.yaml new file mode 100644 index 0000000000..8d723fbd4b --- /dev/null +++ b/oap-server/server-starter/src/main/resources/otel-oc-rules/k8s-instance.yaml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filter: "{ tags -> tags.job_name in [ 'kubernetes-cadvisor', 'kube-state-metrics' ] }" # The OpenTelemetry job name +expSuffix: |- + service(['cluster' , 'service'], '::', Layer.K8S_SERVICE) + .instance(['cluster', 'service'], '::', ['pod'], '', Layer.K8S_SERVICE, { tags -> ['pod': tags.pod, 'namespace': tags.namespace] }) +metricPrefix: k8s_service_instance +metricsRules: + - name: pod_instance_status + exp: kube_pod_status_phase.retagByK8sMeta('service' , K8sRetagType.Pod2Service , 'pod' , 'namespace').tagNotEqual('service' , '').sum(['cluster', 'namespace', 'service' , 'pod'])
