This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch perf/groovy in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit e6682df93bb7933c10ed203702f3067a65b03507 Author: kezhenxu94 <[email protected]> AuthorDate: Wed Jul 7 18:13:37 2021 +0800 perf: optimize Groovy-based DSL with static compilation Groovy naturally supports many dynamic features that we don't benefit for now but cost performance loss, in this patch we compile our Groovy-based DSL scripts statically to optimize performance. --- oap-server/analyzer/log-analyzer/pom.xml | 4 + .../skywalking/oap/log/analyzer/dsl/DSL.java | 17 ++- .../log/analyzer/dsl/LALPrecompiledExtension.java | 89 +++++++++++ .../log/analyzer/dsl/spec/LALDelegatingScript.java | 56 +++++++ .../analyzer/dsl/spec/extractor/ExtractorSpec.java | 29 ++-- .../log/analyzer/dsl/spec/filter/FilterSpec.java | 29 ++-- .../dsl/spec/parser/AbstractParserSpec.java | 18 ++- .../analyzer/dsl/spec/parser/JsonParserSpec.java | 17 +-- .../analyzer/dsl/spec/parser/TextParserSpec.java | 9 +- .../log/analyzer/dsl/spec/sink/SamplerSpec.java | 6 +- .../oap/log/analyzer/dsl/spec/sink/SinkSpec.java | 7 +- .../skywalking/oap/log/analyzer/dsl/DSLTest.java | 162 +++++++++++++++++++++ .../test/resources/log-mal-rules/placeholder.yaml | 20 +-- .../apache/skywalking/oal/rt/grammar/OALLexer.g4 | 2 +- .../agent/kafka/mock/MockModuleManager.java | 2 + test/e2e/e2e-test/docker/log/lal.yaml | 2 +- 16 files changed, 395 insertions(+), 74 deletions(-) diff --git a/oap-server/analyzer/log-analyzer/pom.xml b/oap-server/analyzer/log-analyzer/pom.xml index f1dd346..fbe0e6c 100644 --- a/oap-server/analyzer/log-analyzer/pom.xml +++ b/oap-server/analyzer/log-analyzer/pom.xml @@ -42,6 +42,10 @@ <groupId>org.codehaus.groovy</groupId> <artifactId>groovy</artifactId> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> </dependencies> </project> diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java index c5fad4e..c7db405 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java @@ -19,14 +19,20 @@ package org.apache.skywalking.oap.log.analyzer.dsl; import groovy.lang.GroovyShell; +import groovy.transform.CompileStatic; import groovy.util.DelegatingScript; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.log.analyzer.dsl.spec.LALDelegatingScript; import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec; import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.codehaus.groovy.control.CompilerConfiguration; +import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer; + +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public class DSL { @@ -38,7 +44,16 @@ public class DSL { final LogAnalyzerModuleConfig config, final String dsl) throws ModuleStartException { final CompilerConfiguration cc = new CompilerConfiguration(); - cc.setScriptBaseClass(DelegatingScript.class.getName()); + final ASTTransformationCustomizer customizer = + new ASTTransformationCustomizer( + singletonMap( + "extensions", + singletonList(LALPrecompiledExtension.class.getName()) + ), + CompileStatic.class + ); + cc.addCompilationCustomizers(customizer); + cc.setScriptBaseClass(LALDelegatingScript.class.getName()); final GroovyShell sh = new GroovyShell(cc); final DelegatingScript script = (DelegatingScript) sh.parse(dsl); diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/LALPrecompiledExtension.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/LALPrecompiledExtension.java new file mode 100644 index 0000000..56888a9 --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/LALPrecompiledExtension.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.log.analyzer.dsl; + +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.apm.network.logging.v3.LogDataBody; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.apm.network.logging.v3.TraceContext; +import org.codehaus.groovy.ast.expr.ConstantExpression; +import org.codehaus.groovy.ast.expr.Expression; +import org.codehaus.groovy.ast.expr.PropertyExpression; +import org.codehaus.groovy.ast.expr.VariableExpression; +import org.codehaus.groovy.transform.stc.AbstractTypeCheckingExtension; +import org.codehaus.groovy.transform.stc.StaticTypeCheckingVisitor; + +import static org.codehaus.groovy.ast.ClassHelper.makeCached; + +public class LALPrecompiledExtension extends AbstractTypeCheckingExtension { + + public LALPrecompiledExtension(final StaticTypeCheckingVisitor typeCheckingVisitor) { + super(typeCheckingVisitor); + } + + @Override + public boolean handleUnresolvedProperty(final PropertyExpression pexp) { + final Expression exp = pexp.getObjectExpression(); + + if (exp.getText().startsWith("parsed")) { + makeDynamic(pexp); + setHandled(true); + return true; + } + + if (exp.getText().startsWith("log")) { + if (handleLogVariable(pexp)) { + return true; + } + } + + return super.handleUnresolvedProperty(pexp); + } + + private boolean handleLogVariable(final PropertyExpression pexp) { + final Expression exp = pexp.getObjectExpression(); + final Expression p = pexp.getProperty(); + + if (exp instanceof VariableExpression) { + final VariableExpression v = (VariableExpression) exp; + if (v.getName().equals("log")) { + storeType(v, makeCached(LogData.Builder.class)); + } + if (p instanceof ConstantExpression) { + final ConstantExpression c = (ConstantExpression) p; + switch (c.getText()) { + case "body": + storeType(pexp, makeCached(LogDataBody.class)); + break; + case "traceContext": + storeType(pexp, makeCached(TraceContext.class)); + break; + case "tags": + storeType(pexp, makeCached(LogTags.class)); + break; + } + } + setHandled(true); + return true; + } + + return false; + } +} + diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/LALDelegatingScript.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/LALDelegatingScript.java new file mode 100644 index 0000000..a057373 --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/LALDelegatingScript.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.log.analyzer.dsl.spec; + +import groovy.lang.Closure; +import groovy.lang.DelegatesTo; +import groovy.util.DelegatingScript; +import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec; + +public class LALDelegatingScript extends DelegatingScript { + @Override + public Object run() { + return null; + } + + public void filter(@DelegatesTo(value = FilterSpec.class, strategy = Closure.DELEGATE_ONLY) Closure<?> closure) { + closure.setDelegate(getDelegate()); + closure.call(); + } + + public void json(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) { + closure.setDelegate(getDelegate()); + closure.call(); + } + + public void text(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) { + closure.setDelegate(getDelegate()); + closure.call(); + } + + public void extractor(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) { + closure.setDelegate(getDelegate()); + closure.call(); + } + + public void sink(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) { + closure.setDelegate(getDelegate()); + closure.call(); + } +} diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java index 49ea7d6..c51f445 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java @@ -21,6 +21,7 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.extractor; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import groovy.lang.Closure; +import groovy.lang.DelegatesTo; import java.util.Collection; import java.util.List; import java.util.Map; @@ -54,7 +55,8 @@ public class ExtractorSpec extends AbstractSpec { final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException { super(moduleManager, moduleConfig); - final MeterSystem meterSystem = moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class); + final MeterSystem meterSystem = + moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class); metricConverts = moduleConfig.malConfigs() .stream() @@ -93,7 +95,7 @@ public class ExtractorSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void tag(final Map<String, Object> kv) { + public void tag(final Map<String, ?> kv) { if (BINDING.get().shouldAbort()) { return; } @@ -108,7 +110,8 @@ public class ExtractorSpec extends AbstractSpec { kv.entrySet() .stream() .filter(it -> isNotBlank(it.getKey())) - .filter(it -> nonNull(it.getValue()) && isNotBlank(Objects.toString(it.getValue()))) + .filter(it -> nonNull(it.getValue()) && + isNotBlank(Objects.toString(it.getValue()))) .map(it -> { final Object val = it.getValue(); String valStr = Objects.toString(val); @@ -176,7 +179,7 @@ public class ExtractorSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void metrics(final Closure<Void> cl) { + public void metrics(@DelegatesTo(SampleBuilder.class) final Closure<?> cl) { if (BINDING.get().shouldAbort()) { return; } @@ -188,8 +191,8 @@ public class ExtractorSpec extends AbstractSpec { metricConverts.forEach(it -> it.toMeter( ImmutableMap.<String, SampleFamily>builder() - .put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build()) - .build() + .put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build()) + .build() )); } @@ -198,11 +201,15 @@ public class ExtractorSpec extends AbstractSpec { private final Sample.SampleBuilder sampleBuilder = Sample.builder(); @SuppressWarnings("unused") - public Sample.SampleBuilder labels(final Map<String, String> labels) { - final Map<String, String> filtered = labels.entrySet() - .stream() - .filter(it -> isNotBlank(it.getKey()) && isNotBlank(it.getValue())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + public Sample.SampleBuilder labels(final Map<String, ?> labels) { + final Map<String, String> filtered = + labels.entrySet() + .stream() + .filter(it -> isNotBlank(it.getKey()) && nonNull(it.getValue())) + .collect( + Collectors.toMap(Map.Entry::getKey, + it -> Objects.toString(it.getValue())) + ); return sampleBuilder.labels(ImmutableMap.copyOf(filtered)); } } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java index 7683bb6..43f86a3 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java @@ -18,10 +18,10 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.filter; -import com.google.gson.reflect.TypeToken; +import com.fasterxml.jackson.core.type.TypeReference; import com.google.protobuf.TextFormat; import groovy.lang.Closure; -import java.lang.reflect.Type; +import groovy.lang.DelegatesTo; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -57,14 +57,14 @@ public class FilterSpec extends AbstractSpec { private final SinkSpec sink; - private final Type parsedType; + private final TypeReference<Map<String, Object>> parsedType; public FilterSpec(final ModuleManager moduleManager, final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException { super(moduleManager, moduleConfig); - parsedType = new TypeToken<Map<String, Object>>() { - }.getType(); + parsedType = new TypeReference<Map<String, Object>>() { + }; factories = Arrays.asList( new RecordAnalysisListener.Factory(moduleManager(), moduleConfig()), @@ -81,7 +81,7 @@ public class FilterSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void text(final Closure<Void> cl) { + public void text(@DelegatesTo(TextParserSpec.class) final Closure<?> cl) { if (BINDING.get().shouldAbort()) { return; } @@ -90,7 +90,7 @@ public class FilterSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void json(final Closure<Void> cl) { + public void json(@DelegatesTo(JsonParserSpec.class) final Closure<?> cl) { if (BINDING.get().shouldAbort()) { return; } @@ -99,7 +99,8 @@ public class FilterSpec extends AbstractSpec { final LogData.Builder logData = BINDING.get().log(); try { - final Map<String, Object> parsed = jsonParser.create().fromJson( + + final Map<String, Object> parsed = jsonParser.create().readValue( logData.getBody().getJson().getJson(), parsedType ); @@ -111,8 +112,8 @@ public class FilterSpec extends AbstractSpec { } } - @SuppressWarnings({"unused", "unchecked"}) - public void yaml(final Closure<Void> cl) { + @SuppressWarnings({"unused"}) + public void yaml(@DelegatesTo(YamlParserSpec.class) final Closure<?> cl) { if (BINDING.get().shouldAbort()) { return; } @@ -121,7 +122,7 @@ public class FilterSpec extends AbstractSpec { final LogData.Builder logData = BINDING.get().log(); try { - final Map<String, Object> parsed = (Map<String, Object>) yamlParser.create().load( + final Map<String, Object> parsed = yamlParser.create().load( logData.getBody().getYaml().getYaml() ); @@ -134,7 +135,7 @@ public class FilterSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void extractor(final Closure<Void> cl) { + public void extractor(@DelegatesTo(ExtractorSpec.class) final Closure<?> cl) { if (BINDING.get().shouldAbort()) { return; } @@ -143,7 +144,7 @@ public class FilterSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void sink(final Closure<Void> cl) { + public void sink(@DelegatesTo(SinkSpec.class) final Closure<?> cl) { if (BINDING.get().shouldAbort()) { return; } @@ -166,7 +167,7 @@ public class FilterSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void filter(final Closure<Void> cl) { + public void filter(final Closure<?> cl) { cl.call(); } } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java index e410cbe..a7db119 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java @@ -18,26 +18,32 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser; -import lombok.Getter; -import lombok.Setter; import lombok.experimental.Accessors; import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec; import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleManager; -@Accessors(fluent = true) +@Accessors public class AbstractParserSpec extends AbstractSpec { /** * Whether the filter chain should abort when parsing the logs failed. * - * Failing to parse the logs means either parsing throws exceptions or the logs not matching the desired patterns. + * Failing to parse the logs means either parsing throws exceptions or the logs not matching the + * desired patterns. */ - @Getter - @Setter private boolean abortOnFailure = true; public AbstractParserSpec(final ModuleManager moduleManager, final LogAnalyzerModuleConfig moduleConfig) { super(moduleManager, moduleConfig); } + + @SuppressWarnings("unused") // used in user LAL scripts + public void abortOnFailure(final boolean abortOnFailure) { + this.abortOnFailure = abortOnFailure; + } + + public boolean abortOnFailure() { + return this.abortOnFailure; + } } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java index 850daab..1fabdbc 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java @@ -18,28 +18,23 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleManager; public class JsonParserSpec extends AbstractParserSpec { - private final GsonBuilder gsonBuilder; - - private final Gson gson; + private final ObjectMapper mapper; public JsonParserSpec(final ModuleManager moduleManager, final LogAnalyzerModuleConfig moduleConfig) { super(moduleManager, moduleConfig); - gsonBuilder = new GsonBuilder(); - - // We just create a gson instance in advance for now (for the sake of performance), + // We just create a mapper instance in advance for now (for the sake of performance), // when we want to provide some extra options, we'll move this into method "create" then. - gson = gsonBuilder.create(); + mapper = new ObjectMapper(); } - public Gson create() { - return gson; + public ObjectMapper create() { + return mapper; } } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java index c77f639..f20a1e9 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java @@ -31,13 +31,13 @@ public class TextParserSpec extends AbstractParserSpec { } @SuppressWarnings("unused") - public boolean regexp(final String regexp) { - return regexp(Pattern.compile(regexp)); + public void regexp(final String regexp) { + regexp(Pattern.compile(regexp)); } - public boolean regexp(final Pattern pattern) { + public void regexp(final Pattern pattern) { if (BINDING.get().shouldAbort()) { - return false; + return; } final LogData.Builder log = BINDING.get().log(); final Matcher matcher = pattern.matcher(log.getBody().getText().getText()); @@ -47,7 +47,6 @@ public class TextParserSpec extends AbstractParserSpec { } else if (abortOnFailure()) { BINDING.get().abort(); } - return matched; } public boolean grok(final String grok) { diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java index e32f792..77df935 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java @@ -19,6 +19,8 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.sink; import groovy.lang.Closure; +import groovy.lang.DelegatesTo; +import groovy.lang.GString; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec; @@ -28,7 +30,7 @@ import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleManager; public class SamplerSpec extends AbstractSpec { - private final Map<String, Sampler> samplers; + private final Map<GString, Sampler> samplers; private final RateLimitingSampler.ResetHandler rlsResetHandler; public SamplerSpec(final ModuleManager moduleManager, @@ -40,7 +42,7 @@ public class SamplerSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void rateLimit(final String id, final Closure<Void> cl) { + public void rateLimit(final GString id, @DelegatesTo(RateLimitingSampler.class) final Closure<?> cl) { if (BINDING.get().shouldAbort()) { return; } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java index ee78be9..82566f9 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java @@ -19,6 +19,7 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.sink; import groovy.lang.Closure; +import groovy.lang.DelegatesTo; import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec; import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleManager; @@ -35,7 +36,7 @@ public class SinkSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void sampler(final Closure<Void> cl) { + public void sampler(@DelegatesTo(SamplerSpec.class) final Closure<?> cl) { if (BINDING.get().shouldAbort()) { return; } @@ -44,7 +45,7 @@ public class SinkSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void enforcer(final Closure<Void> cl) { + public void enforcer(final Closure<?> cl) { if (BINDING.get().shouldAbort()) { return; } @@ -52,7 +53,7 @@ public class SinkSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void dropper(final Closure<Void> cl) { + public void dropper(final Closure<?> cl) { if (BINDING.get().shouldAbort()) { return; } diff --git a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java new file mode 100644 index 0000000..8a923ce --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.log.analyzer.dsl; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.ConfigService; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.powermock.reflect.Whitebox; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(Parameterized.class) +public class DSLTest { + @Parameterized.Parameters(name = "{index}: {0}") + public static Collection<Object[]> data() { + return Arrays.asList( + new String[] { + "parser", + "filter {\n" + + " json {\n" + + " abortOnFailure false // for test purpose, we want to persist all logs\n" + + " }\n" + + " text {\n" + + " abortOnFailure false // for test purpose, we want to persist all logs\n" + + " regexp $/(?s)(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) \\[TID:(?<tid>.+?)] \\[(?<thread>.+?)] (?<level>\\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$" + + " }\n" + + " yaml {\n" + + " abortOnFailure false // for test purpose, we want to persist all logs\n" + + " }" + + "}", + }, + new String[] { + "extractor", + "filter {\n" + + " extractor {\n" + + " service \"test\"\n" + + " instance \"test\"\n" + + " endpoint \"test\"\n" + + " traceId \"123\"\n" + + " segmentId \"123\"\n" + + " spanId \"123\"\n" + + " timestamp \"123\"\n" + + " metrics {\n" + + " name \"metricsName\"\n" + + " value 123\n" + + " timestamp \"123\"\n" + + " labels \"k1\": \"v1\"\n" + + " }\n" + + " }\n" + + "}", + }, + new String[] { + "sink", + "filter {\n" + + " sink {\n" + + " enforcer {\n" + + " }\n" + + " dropper {\n" + + " }\n" + + " sampler {\n" + + " if (parsed?.commonProperties?.responseFlags) {\n" + + " // use service:errorCode as sampler id so that each service:errorCode has its own sampler,\n" + + " // e.g. checkoutservice:[upstreamConnectionFailure], checkoutservice:[upstreamRetryLimitExceeded]\n" + + " rateLimit(\"${log.service}:${log.body.json.json}:${log.tags.getData(0).key}:${parsed?.commonProperties?.responseFlags}\") {\n" + + " qps 100\n" + + " }\n" + + " } else {\n" + + " // use service:responseCode as sampler id so that each service:responseCode has its own sampler,\n" + + " // e.g. checkoutservice:500, checkoutservice:404.\n" + + " rateLimit(\"${log.service}:${log.body?.type}:${log.traceContext?.traceId}:${parsed?.response?.responseCode}\") {\n" + + " qps 100\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", + }, + new String[] { + "e2e", + "filter {\n" + + " text {\n" + + " abortOnFailure false // for test purpose, we want to persist all logs\n" + + " regexp $/(?s)(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) \\[TID:(?<tid>.+?)] \\[(?<thread>.+?)] (?<level>\\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$\n" + + " }\n" + + " extractor {\n" + + " metrics {\n" + + " timestamp \"${log.timestamp}\"\n" + + " labels level: parsed.level, service: log.service, instance: log.serviceInstance\n" + + " name \"log_count\"\n" + + " value 1\n" + + " }\n" + + " }\n" + + " sink {\n" + + " }\n" + + "}\n" + } + ); + } + + @Parameterized.Parameter() + public String name; + + @Parameterized.Parameter(1) + public String script; + + final ModuleManager manager = mock(ModuleManager.class); + + @Before + public void setup() { + Whitebox.setInternalState(manager, "isInPrepareStage", false); + when(manager.find(anyString())).thenReturn(mock(ModuleProviderHolder.class)); + when(manager.find(CoreModule.NAME).provider()).thenReturn(mock(ModuleServiceHolder.class)); + when(manager.find(CoreModule.NAME).provider().getService(SourceReceiver.class)) + .thenReturn(mock(SourceReceiver.class)); + when(manager.find(CoreModule.NAME).provider().getService( + ConfigService.class)).thenReturn(mock(ConfigService.class)); + when(manager.find(CoreModule.NAME).provider().getService( + ConfigService.class).getSearchableLogsTags()).thenReturn(anyString()); + } + + @Test + public void testDslStaticCompile() throws ModuleStartException { + final DSL dsl = DSL.of(manager, new LogAnalyzerModuleConfig(), script); + Whitebox.setInternalState( + Whitebox.getInternalState(dsl, "filterSpec"), "factories", Collections.emptyList() + ); + + dsl.bind(new Binding().log(LogData.newBuilder().build())); + dsl.evaluate(); + } +} diff --git a/test/e2e/e2e-test/docker/log/lal.yaml b/oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml similarity index 56% copy from test/e2e/e2e-test/docker/log/lal.yaml copy to oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml index 2b94988..034440c 100644 --- a/test/e2e/e2e-test/docker/log/lal.yaml +++ b/oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml @@ -13,22 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -rules: - - name: example - dsl: | - filter { - text { - abortOnFailure false // for test purpose, we want to persist all logs - regexp $/(?s)(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}) \[TID:(?<tid>.+?)] \[(?<thread>.+?)] (?<level>\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$ - } - extractor { - metrics { - timestamp log.timestamp - labels level: parsed.level, service: log.service, instance: log.serviceInstance - name "log_count" - value 1 - } - } - sink { - } - } +# Refer to examples in config-examples/log-mal.yaml diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 index 25654da..b881a98 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 @@ -39,7 +39,7 @@ SRC_SERVICE_INSTANCE_JVM_MEMORY: 'ServiceInstanceJVMMemory'; SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL: 'ServiceInstanceJVMMemoryPool'; SRC_SERVICE_INSTANCE_JVM_GC: 'ServiceInstanceJVMGC'; SRC_SERVICE_INSTANCE_JVM_THREAD: 'ServiceInstanceJVMThread'; -SRC_SERVICE_INSTANCE_JVM_CLASS:'ServiceInstanceJVMClass'; +SRC_SERVICE_INSTANCE_JVM_CLASS: 'ServiceInstanceJVMClass'; SRC_DATABASE_ACCESS: 'DatabaseAccess'; SRC_SERVICE_INSTANCE_CLR_CPU: 'ServiceInstanceCLRCPU'; SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC'; diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java index 8d5ee8a..63c95bb 100644 --- a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java @@ -37,10 +37,12 @@ public abstract class MockModuleManager extends ModuleManager { moduleProviderHolderMap.put(name, provider); } + @Override public boolean has(String moduleName) { return moduleProviderHolderMap.containsKey(moduleName); } + @Override public ModuleProviderHolder find(String moduleName) throws ModuleNotFoundRuntimeException { if (!moduleProviderHolderMap.containsKey(moduleName)) { throw new ModuleNotFoundRuntimeException("ModuleProviderHolder[" + moduleName + "] cannot found in MOCK."); diff --git a/test/e2e/e2e-test/docker/log/lal.yaml b/test/e2e/e2e-test/docker/log/lal.yaml index 2b94988..bfd7a61 100644 --- a/test/e2e/e2e-test/docker/log/lal.yaml +++ b/test/e2e/e2e-test/docker/log/lal.yaml @@ -23,7 +23,7 @@ rules: } extractor { metrics { - timestamp log.timestamp + timestamp log.timestamp as Long labels level: parsed.level, service: log.service, instance: log.serviceInstance name "log_count" value 1
