This is an automated email from the ASF dual-hosted git repository. wusheng pushed a commit to branch fix/lal-output-builder-cleanup in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 5d4fbb93345b7dd0387549dfae5c8ba6e9bc3332 Author: Wu Sheng <[email protected]> AuthorDate: Tue Mar 10 14:54:09 2026 +0800 LAL: generalize extraLog type, add EnvoyAccessLogBuilder, compile-time tag validation, and DSL class generator test --- .../operation/dynamic-code-generation-debugging.md | 37 ++ .../listener/DatabaseSlowStatementBuilder.java | 3 +- .../trace/parser/listener/SampledTraceBuilder.java | 3 +- .../log/analyzer/v2/compiler/LALBlockCodegen.java | 12 + .../oap/log/analyzer/v2/dsl/ExecutionContext.java | 7 +- .../analyzer/v2/dsl/spec/filter/FilterSpec.java | 4 +- .../provider/log/ILogAnalysisListenerManager.java | 5 - .../v2/provider/log/ILogAnalyzerService.java | 6 +- .../log/analyzer/v2/provider/log/LogAnalyzer.java | 19 +- .../v2/provider/log/LogAnalyzerServiceImpl.java | 18 +- .../provider/log/listener/LogAnalysisListener.java | 8 +- .../provider/log/listener/LogFilterListener.java | 28 +- .../v2/provider/log/listener/LogSinkListener.java | 6 +- .../provider/log/listener/RecordSinkListener.java | 10 +- .../provider/log/listener/TrafficSinkListener.java | 4 +- .../log/analyzer/v2/spi/LALSourceTypeProvider.java | 6 +- .../oap/server/core/source/LALOutputBuilder.java | 23 +- .../oap/server/core/source/LogBuilder.java | 4 +- .../envoy/EnvoyHTTPLALSourceTypeProvider.java | 10 + .../envoy/persistence/EnvoyAccessLogBuilder.java | 73 +++ .../envoy/persistence/LogsPersistence.java | 3 +- .../envoy/persistence/TCPLogsPersistence.java | 3 +- ...walking.oap.server.core.source.LALOutputBuilder | 18 + .../persistence/EnvoyAccessLogBuilderTest.java | 224 +++++++ .../otel/otlp/OpenTelemetryLogHandler.java | 3 +- .../handler/grpc/LogReportServiceGrpcHandler.java | 3 +- .../handler/rest/LogReportServiceHTTPHandler.java | 3 +- .../oap/server/starter/DSLClassGeneratorTest.java | 653 +++++++++++++++++++++ 28 files changed, 1118 insertions(+), 78 deletions(-) diff --git a/docs/en/operation/dynamic-code-generation-debugging.md b/docs/en/operation/dynamic-code-generation-debugging.md index 15351fce84..b582f81df1 100644 --- a/docs/en/operation/dynamic-code-generation-debugging.md +++ b/docs/en/operation/dynamic-code-generation-debugging.md @@ -150,6 +150,43 @@ Reading this: 4. **Use the statement number** (after the last `:`) as a rough indicator of which operation within the generated method failed. Dump the class (see above) and use `javap -v` to see the exact mapping. +## Generating All DSL Classes Offline + +You can compile every DSL script from the source tree without starting the OAP server. +This is useful for verifying that all scripts are syntactically valid after editing, or for +batch-inspecting the generated bytecode with `javap` or an IDE decompiler. + +The `DSLClassGeneratorTest` in the `server-starter` module compiles all OAL, MAL, LAL, and +Hierarchy scripts and dumps the `.class` files to `target/generated-dsl-classes/`. + +```shell +# From the project root (requires a prior build: ./mvnw -Pbackend install -Dmaven.test.skip) +./mvnw test -pl oap-server/server-starter \ + -Dtest="DSLClassGeneratorTest#generateAllDSLClasses" \ + -Dcheckstyle.skip +``` + +### Output + +``` +oap-server/server-starter/target/generated-dsl-classes/ +├── oal/ ← Metrics, MetricsBuilder, Dispatcher classes +├── mal/ ← MAL expression and filter classes +├── lal/ ← LAL expression classes +└── hierarchy/ ← Hierarchy rule classes +``` + +The test fails if any script cannot be compiled and prints the list of failures. + +### What It Covers + +| DSL | Scripts | Source Directory | +|-----|---------|-----------------| +| OAL | All 9 `.oal` files (core, java-agent, dotnet-agent, browser, mesh, tcp, ebpf, cilium, disable) | `src/main/resources/oal/` | +| MAL | All YAML files across 4 directories | `src/main/resources/{otel-rules,meter-analyzer-config,envoy-metrics-rules,log-mal-rules}/` | +| LAL | All YAML files, with SPI-resolved `inputType`/`outputType` | `src/main/resources/lal/` | +| Hierarchy | All `auto-matching-rules` entries | `src/main/resources/hierarchy-definition.yml` | + ## Common Error Patterns ### OAL Compilation Failure diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/DatabaseSlowStatementBuilder.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/DatabaseSlowStatementBuilder.java index 2ff8aab248..654afcab8a 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/DatabaseSlowStatementBuilder.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/DatabaseSlowStatementBuilder.java @@ -75,7 +75,8 @@ public class DatabaseSlowStatementBuilder implements LALOutputBuilder { } @Override - public void init(final LogData logData, final NamingControl namingControl) { + public void init(final Object logDataObj, final NamingControl namingControl) { + final LogData logData = (LogData) logDataObj; this.namingControl = namingControl; // Only populate fields not already set by the LAL extractor. if (this.serviceName == null) { diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SampledTraceBuilder.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SampledTraceBuilder.java index 6952fe5939..171232f418 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SampledTraceBuilder.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SampledTraceBuilder.java @@ -99,7 +99,8 @@ public class SampledTraceBuilder implements LALOutputBuilder { } @Override - public void init(final LogData logData, final NamingControl namingControl) { + public void init(final Object logDataObj, final NamingControl namingControl) { + final LogData logData = (LogData) logDataObj; this.namingControl = namingControl; // Only populate fields not already set by the LAL extractor. if (this.traceId == null) { diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALBlockCodegen.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALBlockCodegen.java index a51249c0a0..3d10117f00 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALBlockCodegen.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/compiler/LALBlockCodegen.java @@ -267,6 +267,18 @@ final class LALBlockCodegen { static void generateTagAssignment(final StringBuilder sb, final LALScriptModel.TagAssignment tag, final LALClassGenerator.GenCtx genCtx) { + // tag assignments are only supported on LogBuilder (the default output type). + // Other output types (e.g. SampledTraceBuilder, DatabaseSlowStatementBuilder) + // do not carry tags — fail at compile time instead of silently dropping them. + if (genCtx.outputType != null + && !org.apache.skywalking.oap.server.core.source.LogBuilder.class + .isAssignableFrom(genCtx.outputType)) { + throw new IllegalArgumentException( + "LAL 'tag' assignments are only supported when outputType is LogBuilder" + + " (or default), but the resolved outputType is " + + genCtx.outputType.getName() + + ". Remove the 'tag' statements or change the outputType."); + } for (final Map.Entry<String, LALScriptModel.TagValue> entry : tag.getTags().entrySet()) { sb.append(" _o.addTag(\"") diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/ExecutionContext.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/ExecutionContext.java index 913a91a3e0..992d797720 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/ExecutionContext.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/ExecutionContext.java @@ -17,7 +17,6 @@ package org.apache.skywalking.oap.log.analyzer.v2.dsl; -import com.google.protobuf.Message; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -85,12 +84,12 @@ public class ExecutionContext { return (LogData.Builder) getProperty(KEY_LOG); } - public ExecutionContext extraLog(final Message extraLog) { + public ExecutionContext extraLog(final Object extraLog) { parsed().extraLog = extraLog; return this; } - public Message extraLog() { + public Object extraLog() { return parsed().getExtraLog(); } @@ -179,6 +178,6 @@ public class ExecutionContext { private Map<String, Object> map; @Getter - private Message extraLog; + private Object extraLog; } } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/spec/filter/FilterSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/spec/filter/FilterSpec.java index 730caa0630..476a9802f4 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/spec/filter/FilterSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/dsl/spec/filter/FilterSpec.java @@ -19,11 +19,11 @@ package org.apache.skywalking.oap.log.analyzer.v2.dsl.spec.filter; import com.fasterxml.jackson.core.type.TypeReference; -import com.google.protobuf.Message; import com.google.protobuf.TextFormat; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.oap.log.analyzer.v2.dsl.ExecutionContext; import org.apache.skywalking.oap.log.analyzer.v2.dsl.spec.AbstractSpec; @@ -172,7 +172,7 @@ public class FilterSpec extends AbstractSpec { private void doSink(final ExecutionContext ctx) { final LogData.Builder logData = ctx.log(); - final Message extraLog = ctx.extraLog(); + final Optional<Object> extraLog = Optional.ofNullable(ctx.extraLog()); if (!ctx.shouldSave()) { if (LOGGER.isDebugEnabled()) { diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/ILogAnalysisListenerManager.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/ILogAnalysisListenerManager.java index 421ed6edbf..9825beb291 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/ILogAnalysisListenerManager.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/ILogAnalysisListenerManager.java @@ -19,15 +19,10 @@ package org.apache.skywalking.oap.log.analyzer.v2.provider.log; import java.util.List; import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.LogAnalysisListenerFactory; -import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.LogSinkListenerFactory; public interface ILogAnalysisListenerManager { void addListenerFactory(LogAnalysisListenerFactory factory); List<LogAnalysisListenerFactory> getLogAnalysisListenerFactories(); - - void addSinkListenerFactory(LogSinkListenerFactory factory); - - List<LogSinkListenerFactory> getSinkListenerFactory(); } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/ILogAnalyzerService.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/ILogAnalyzerService.java index 8498f73aa6..ec951fba50 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/ILogAnalyzerService.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/ILogAnalyzerService.java @@ -17,7 +17,7 @@ package org.apache.skywalking.oap.log.analyzer.v2.provider.log; -import com.google.protobuf.Message; +import java.util.Optional; import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.oap.server.library.module.Service; @@ -26,9 +26,9 @@ import org.apache.skywalking.oap.server.library.module.Service; */ public interface ILogAnalyzerService extends Service { - void doAnalysis(LogData.Builder log, Message extraLog); + void doAnalysis(LogData.Builder log, Optional<Object> extraLog); - default void doAnalysis(LogData logData, Message extraLog) { + default void doAnalysis(LogData logData, Optional<Object> extraLog) { doAnalysis(logData.toBuilder(), extraLog); } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzer.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzer.java index 6ae2c312ec..08f84a9156 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzer.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzer.java @@ -17,20 +17,16 @@ package org.apache.skywalking.oap.log.analyzer.v2.provider.log; -import com.google.protobuf.Message; import java.util.ArrayList; import java.util.List; import java.util.Objects; - -import lombok.RequiredArgsConstructor; +import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.LogAnalysisListener; import org.apache.skywalking.oap.server.core.UnexpectedException; import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.library.util.StringUtil; -import org.apache.skywalking.oap.log.analyzer.v2.provider.LogAnalyzerModuleConfig; -import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.LogAnalysisListener; -import org.apache.skywalking.oap.server.library.module.ModuleManager; /** * Entry point for log analysis. Created per-request by the log receiver. @@ -53,15 +49,16 @@ import org.apache.skywalking.oap.server.library.module.ModuleManager; * </ol> */ @Slf4j -@RequiredArgsConstructor public class LogAnalyzer { - private final ModuleManager moduleManager; - private final LogAnalyzerModuleConfig moduleConfig; private final ILogAnalysisListenerManager factoryManager; + public LogAnalyzer(final ILogAnalysisListenerManager factoryManager) { + this.factoryManager = factoryManager; + } + private final List<LogAnalysisListener> listeners = new ArrayList<>(); - public void doAnalysis(LogData.Builder builder, Message extraLog) { + public void doAnalysis(LogData.Builder builder, Optional<Object> extraLog) { if (StringUtil.isEmpty(builder.getService())) { // If the service name is empty, the log will be ignored. log.debug("The log is ignored because the Service name is empty"); @@ -89,7 +86,7 @@ public class LogAnalyzer { notifyAnalysisListenerToBuild(); } - private void notifyAnalysisListener(LogData.Builder builder, final Message extraLog) { + private void notifyAnalysisListener(LogData.Builder builder, final Optional<Object> extraLog) { listeners.forEach(listener -> listener.parse(builder, extraLog)); } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzerServiceImpl.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzerServiceImpl.java index 8ab5dc0db7..8e1c746bfc 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzerServiceImpl.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/LogAnalyzerServiceImpl.java @@ -17,14 +17,13 @@ package org.apache.skywalking.oap.log.analyzer.v2.provider.log; -import com.google.protobuf.Message; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import lombok.RequiredArgsConstructor; import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.oap.log.analyzer.v2.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.LogAnalysisListenerFactory; -import org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener.LogSinkListenerFactory; import org.apache.skywalking.oap.server.library.module.ModuleManager; @RequiredArgsConstructor @@ -32,11 +31,10 @@ public class LogAnalyzerServiceImpl implements ILogAnalyzerService, ILogAnalysis private final ModuleManager moduleManager; private final LogAnalyzerModuleConfig moduleConfig; private final List<LogAnalysisListenerFactory> analysisListenerFactories = new ArrayList<>(); - private final List<LogSinkListenerFactory> sinkListenerFactories = new ArrayList<>(); @Override - public void doAnalysis(final LogData.Builder log, Message extraLog) { - LogAnalyzer analyzer = new LogAnalyzer(moduleManager, moduleConfig, this); + public void doAnalysis(final LogData.Builder log, Optional<Object> extraLog) { + LogAnalyzer analyzer = new LogAnalyzer(this); analyzer.doAnalysis(log, extraLog); } @@ -49,14 +47,4 @@ public class LogAnalyzerServiceImpl implements ILogAnalyzerService, ILogAnalysis public List<LogAnalysisListenerFactory> getLogAnalysisListenerFactories() { return analysisListenerFactories; } - - @Override - public void addSinkListenerFactory(LogSinkListenerFactory factory) { - sinkListenerFactories.add(factory); - } - - @Override - public List<LogSinkListenerFactory> getSinkListenerFactory() { - return sinkListenerFactories; - } } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogAnalysisListener.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogAnalysisListener.java index 9e7fd833b5..fe299835e9 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogAnalysisListener.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogAnalysisListener.java @@ -17,7 +17,7 @@ package org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener; -import com.google.protobuf.Message; +import java.util.Optional; import org.apache.skywalking.apm.network.logging.v3.LogData; /** @@ -31,7 +31,11 @@ public interface LogAnalysisListener { /** * Parse the raw data from the probe. + * @param logData log metadata (service, layer, timestamp, etc.) + * @param extraLog the actual input object whose type matches + * {@code LALSourceTypeProvider#inputType()} — may be {@code null} + * for standard logs where LogData is the sole input * @return {@code this} for chaining. */ - LogAnalysisListener parse(LogData.Builder logData, final Message extraLog); + LogAnalysisListener parse(LogData.Builder logData, Optional<Object> extraLog); } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListener.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListener.java index 954fffdf1e..0d1873ac91 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListener.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogFilterListener.java @@ -18,11 +18,11 @@ package org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener; -import com.google.protobuf.Message; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.ServiceLoader; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -38,7 +38,6 @@ import org.apache.skywalking.oap.log.analyzer.v2.spi.LALSourceTypeProvider; import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.source.LALOutputBuilder; -import org.apache.skywalking.oap.server.core.source.Source; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.module.ModuleStartException; @@ -78,15 +77,34 @@ public class LogFilterListener implements LogAnalysisListener { @Override public LogAnalysisListener parse(final LogData.Builder logData, - final Message extraLog) { + final Optional<Object> extraLog) { final LogData logDataSnapshot = logData.build(); contexts = new ArrayList<>(dsls.size()); for (int i = 0; i < dsls.size(); i++) { - contexts.add(new ExecutionContext().log(logDataSnapshot).extraLog(extraLog)); + contexts.add(new ExecutionContext().log(logDataSnapshot).extraLog(extraLog.orElse(null))); } return this; } + /** + * Eagerly compiles all LAL rules at startup and groups the resulting + * {@link DSL} instances by telemetry layer and rule name. + * + * <p>{@code dsls} structure: {@code Layer -> (ruleName -> DSL)}. + * <ul> + * <li><b>Outer key</b> ({@link Layer}): the telemetry layer declared in + * the YAML rule (e.g., {@code GENERAL}, {@code MESH}).</li> + * <li><b>Inner key</b> ({@code String}): the rule {@code name} field + * from the YAML config (e.g., {@code "default"}, {@code "envoy-als"}). + * Must be unique within a layer.</li> + * <li><b>Value</b> ({@link DSL}): a compiled LAL expression plus its + * runtime {@link org.apache.skywalking.oap.log.analyzer.v2.dsl.spec.filter.FilterSpec}, + * ready to evaluate incoming logs.</li> + * </ul> + * + * <p>At runtime, {@link #create(Layer)} returns a {@link LogFilterListener} + * containing all DSL instances for the requested layer. + */ public static class Factory implements LogAnalysisListenerFactory { private final Map<Layer, Map<String, DSL>> dsls; @@ -194,7 +212,7 @@ public class LogFilterListener implements LogAnalysisListener { } // Fall back to SPI default for the layer if (spiProvider != null) { - final Class<? extends Source> spiOutput = spiProvider.outputType(); + final Class<?> spiOutput = spiProvider.outputType(); if (spiOutput != null) { return spiOutput; } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogSinkListener.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogSinkListener.java index 6ace505bd7..afda6f0671 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogSinkListener.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/LogSinkListener.java @@ -17,7 +17,7 @@ package org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener; -import com.google.protobuf.Message; +import java.util.Optional; import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.oap.log.analyzer.v2.dsl.ExecutionContext; @@ -32,7 +32,7 @@ public interface LogSinkListener { * Parse the raw data from the probe. * @return {@code this} for chaining. */ - LogSinkListener parse(LogData.Builder logData, final Message extraLog); + LogSinkListener parse(LogData.Builder logData, Optional<Object> extraLog); /** * Parse the raw data from the probe with access to the execution context. @@ -40,7 +40,7 @@ public interface LogSinkListener { * per-execution state to the sink output. * @return {@code this} for chaining. */ - default LogSinkListener parse(final LogData.Builder logData, final Message extraLog, + default LogSinkListener parse(final LogData.Builder logData, final Optional<Object> extraLog, final ExecutionContext ctx) { return parse(logData, extraLog); } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/RecordSinkListener.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/RecordSinkListener.java index 77d4154c2d..adda89e416 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/RecordSinkListener.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/RecordSinkListener.java @@ -17,9 +17,9 @@ package org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener; -import com.google.protobuf.Message; import java.util.Arrays; import java.util.List; +import java.util.Optional; import lombok.SneakyThrows; import org.apache.skywalking.apm.network.logging.v3.LogData; @@ -74,14 +74,14 @@ public class RecordSinkListener implements LogSinkListener { @Override @SneakyThrows public LogSinkListener parse(final LogData.Builder logData, - final Message extraLog) { + final Optional<Object> extraLog) { return this; } @Override @SneakyThrows public LogSinkListener parse(final LogData.Builder logData, - final Message extraLog, + final Optional<Object> extraLog, final ExecutionContext ctx) { if (ctx == null || !(ctx.output() instanceof LALOutputBuilder)) { return this; @@ -90,7 +90,9 @@ public class RecordSinkListener implements LogSinkListener { if (builder instanceof LogBuilder) { ((LogBuilder) builder).setSearchableTagKeys(searchableTagKeys); } - builder.init(logData.build(), namingControl); + // Pass the input data matching the declared inputType: + // extraLog (e.g., HTTPAccessLogEntry) when present, otherwise LogData. + builder.init(extraLog.orElseGet(logData::build), namingControl); return this; } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/TrafficSinkListener.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/TrafficSinkListener.java index 49d426d601..8a9ab2b6f2 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/TrafficSinkListener.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/provider/log/listener/TrafficSinkListener.java @@ -17,7 +17,7 @@ package org.apache.skywalking.oap.log.analyzer.v2.provider.log.listener; -import com.google.protobuf.Message; +import java.util.Optional; import lombok.RequiredArgsConstructor; import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.oap.server.core.analysis.Layer; @@ -63,7 +63,7 @@ public class TrafficSinkListener implements LogSinkListener { @Override public LogSinkListener parse(final LogData.Builder logData, - final Message extraLog) { + final Optional<Object> extraLog) { Layer layer; if (StringUtil.isNotEmpty(logData.getLayer())) { layer = Layer.valueOf(logData.getLayer()); diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/spi/LALSourceTypeProvider.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/spi/LALSourceTypeProvider.java index 81b6030553..d843b9159c 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/spi/LALSourceTypeProvider.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/v2/spi/LALSourceTypeProvider.java @@ -18,7 +18,6 @@ package org.apache.skywalking.oap.log.analyzer.v2.spi; import org.apache.skywalking.oap.server.core.analysis.Layer; -import org.apache.skywalking.oap.server.core.source.Source; /** * SPI for receiver plugins to declare the input and default output types for @@ -70,11 +69,12 @@ public interface LALSourceTypeProvider { Class<?> inputType(); /** - * The default {@link Source} subclass that LAL rules on this layer produce. + * The default output type that LAL rules on this layer produce. + * Can be a {@code Source} subclass or an {@code LALOutputBuilder} implementation. * Individual rules can override this via the {@code outputType} YAML config field. * Returns {@code null} by default, meaning the standard {@code Log} source is used. */ - default Class<? extends Source> outputType() { + default Class<?> outputType() { return null; } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LALOutputBuilder.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LALOutputBuilder.java index e0b81085b2..5829dfd466 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LALOutputBuilder.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LALOutputBuilder.java @@ -18,7 +18,6 @@ package org.apache.skywalking.oap.server.core.source; -import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.oap.server.core.config.NamingControl; /** @@ -32,7 +31,11 @@ import org.apache.skywalking.oap.server.core.config.NamingControl; * <p>The LAL compiler validates output field assignments against the builder's * setters at compile time. At runtime: * <ol> - * <li>{@link #init} is called to pre-populate standard fields from LogData</li> + * <li>{@link #init} is called to pre-populate standard fields from the input + * data object whose type matches what + * {@code LALSourceTypeProvider#inputType()} declares (e.g., + * {@code LogData} for standard logs, + * {@code HTTPAccessLogEntry} for envoy access logs)</li> * <li>Output field values are applied via reflection (setter invocation)</li> * <li>{@link #complete} is called to validate, create final Source/Record, * and dispatch via {@link SourceReceiver}</li> @@ -46,10 +49,16 @@ public interface LALOutputBuilder { String name(); /** - * Pre-populate standard fields from the log data before custom output + * Pre-populate standard fields from the input data before custom output * fields are applied. Called once per log entry. + * + * <p>The actual type of {@code logData} matches what the + * {@code LALSourceTypeProvider#inputType()} declares for the layer. + * For standard logs this is {@code LogData}; for envoy access logs + * this is {@code HTTPAccessLogEntry}, etc. Each builder casts directly + * to its expected type. */ - void init(LogData logData, NamingControl namingControl); + void init(Object logData, NamingControl namingControl); /** * Validate the builder state and dispatch the final output source(s). @@ -57,10 +66,4 @@ public interface LALOutputBuilder { */ void complete(SourceReceiver sourceReceiver); - /** - * Add a tag to the output. Only meaningful for builders that produce - * tag-bearing sources (e.g., {@link LogBuilder}). Default is no-op. - */ - default void addTag(final String key, final String value) { - } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogBuilder.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogBuilder.java index 06bc13f8e9..dd4d41a910 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogBuilder.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/LogBuilder.java @@ -77,7 +77,6 @@ public class LogBuilder implements LALOutputBuilder { } } - @Override public void addTag(final String key, final String value) { if (StringUtil.isNotEmpty(key) && StringUtil.isNotEmpty(value)) { lalTags.add(new String[]{key, value}); @@ -90,7 +89,8 @@ public class LogBuilder implements LALOutputBuilder { } @Override - public void init(final LogData logData, final NamingControl namingControl) { + public void init(final Object logDataObj, final NamingControl namingControl) { + final LogData logData = (LogData) logDataObj; this.namingControl = namingControl; this.logData = logData; // Only populate fields that were NOT already set by the LAL extractor. diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyHTTPLALSourceTypeProvider.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyHTTPLALSourceTypeProvider.java index d2a73d568b..df6270e243 100644 --- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyHTTPLALSourceTypeProvider.java +++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyHTTPLALSourceTypeProvider.java @@ -20,11 +20,16 @@ package org.apache.skywalking.oap.server.receiver.envoy; import io.envoyproxy.envoy.data.accesslog.v3.HTTPAccessLogEntry; import org.apache.skywalking.oap.log.analyzer.v2.spi.LALSourceTypeProvider; import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.receiver.envoy.persistence.EnvoyAccessLogBuilder; /** * Declares {@link HTTPAccessLogEntry} as the extra log type for the * {@link Layer#MESH} layer, enabling the LAL compiler to generate direct * proto getter calls for envoy access log rules. + * + * <p>Also declares {@link EnvoyAccessLogBuilder} as the default output type, + * so that the raw access log entry is serialized as JSON content only when + * the log is actually persisted (after LAL filtering). */ public class EnvoyHTTPLALSourceTypeProvider implements LALSourceTypeProvider { @Override @@ -36,4 +41,9 @@ public class EnvoyHTTPLALSourceTypeProvider implements LALSourceTypeProvider { public Class<?> inputType() { return HTTPAccessLogEntry.class; } + + @Override + public Class<?> outputType() { + return EnvoyAccessLogBuilder.class; + } } diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/EnvoyAccessLogBuilder.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/EnvoyAccessLogBuilder.java new file mode 100644 index 0000000000..0ec643ec79 --- /dev/null +++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/EnvoyAccessLogBuilder.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.envoy.persistence; + +import com.google.protobuf.Message; +import lombok.SneakyThrows; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.query.type.ContentType; +import org.apache.skywalking.oap.server.core.source.Log; +import org.apache.skywalking.oap.server.core.source.LogBuilder; +import org.apache.skywalking.oap.server.library.util.ProtoBufJsonUtils; + +/** + * LAL output builder for envoy access logs (both HTTP and TCP). + * + * <p>Extends {@link LogBuilder} to serialize the raw protobuf access log entry + * as JSON content. The serialization is deferred to {@link #toLog()} so that + * it only happens when the log is actually persisted (after LAL filtering). + * + * <p>The {@link #init} method receives the raw protobuf access log entry + * directly (e.g., {@code HTTPAccessLogEntry}) as declared by + * {@code EnvoyHTTPLALSourceTypeProvider#inputType()}, and passes a default + * {@code LogData} to the base class since all standard fields are populated + * by the LAL extractor. + */ +public class EnvoyAccessLogBuilder extends LogBuilder { + public static final String NAME = "EnvoyAccessLog"; + + private Message accessLogEntry; + + @Override + public String name() { + return NAME; + } + + @Override + public void init(final Object logData, final NamingControl namingControl) { + this.accessLogEntry = (Message) logData; + // Standard fields (service, instance, endpoint, layer, etc.) are + // populated by the LAL extractor before init() is called. + // Pass a default LogData to the base class so toLog() body/tag + // accessors work safely (both will be empty for envoy). + super.init(LogData.getDefaultInstance(), namingControl); + } + + @Override + @SneakyThrows + public Log toLog() { + final Log log = super.toLog(); + if (log.getContent() == null && accessLogEntry != null) { + log.setContentType(ContentType.JSON); + log.setContent(ProtoBufJsonUtils.toJSON(accessLogEntry)); + } + return log; + } +} diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/LogsPersistence.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/LogsPersistence.java index e796f889e7..09437cbe3b 100644 --- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/LogsPersistence.java +++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/LogsPersistence.java @@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.receiver.envoy.persistence; import io.envoyproxy.envoy.data.accesslog.v3.HTTPAccessLogEntry; import io.envoyproxy.envoy.service.accesslog.v3.StreamAccessLogsMessage; +import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.apm.network.servicemesh.v3.HTTPServiceMeshMetric; @@ -66,7 +67,7 @@ public class LogsPersistence implements ALSHTTPAnalysis { } final LogData logData = convertToLogData(entry, result); - logAnalyzerService.doAnalysis(logData, entry); + logAnalyzerService.doAnalysis(logData, Optional.of(entry)); } catch (final Exception e) { log.error("Failed to persist Envoy access log", e); } diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/TCPLogsPersistence.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/TCPLogsPersistence.java index 85d91831e5..ac66002d1f 100644 --- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/TCPLogsPersistence.java +++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/TCPLogsPersistence.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.receiver.envoy.persistence; +import java.util.Optional; import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.apm.network.servicemesh.v3.TCPServiceMeshMetric; import org.apache.skywalking.oap.log.analyzer.v2.module.LogAnalyzerModule; @@ -66,7 +67,7 @@ public class TCPLogsPersistence implements TCPAccessLogAnalyzer { } final LogData logData = convertToLogData(entry, result); - logAnalyzerService.doAnalysis(logData, entry); + logAnalyzerService.doAnalysis(logData, Optional.of(entry)); } catch (final Exception e) { log.error("Failed to persist Envoy access log", e); } diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.core.source.LALOutputBuilder b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.core.source.LALOutputBuilder new file mode 100644 index 0000000000..1309ca6d92 --- /dev/null +++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.core.source.LALOutputBuilder @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.skywalking.oap.server.receiver.envoy.persistence.EnvoyAccessLogBuilder diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/EnvoyAccessLogBuilderTest.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/EnvoyAccessLogBuilderTest.java new file mode 100644 index 0000000000..c632b0a12d --- /dev/null +++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/envoy/persistence/EnvoyAccessLogBuilderTest.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.envoy.persistence; + +import com.google.protobuf.UInt32Value; +import io.envoyproxy.envoy.data.accesslog.v3.AccessLogCommon; +import io.envoyproxy.envoy.data.accesslog.v3.HTTPAccessLogEntry; +import io.envoyproxy.envoy.data.accesslog.v3.HTTPResponseProperties; +import java.util.Collections; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.oap.log.analyzer.v2.compiler.LALClassGenerator; +import org.apache.skywalking.oap.log.analyzer.v2.dsl.ExecutionContext; +import org.apache.skywalking.oap.log.analyzer.v2.dsl.LalExpression; +import org.apache.skywalking.oap.log.analyzer.v2.dsl.spec.filter.FilterSpec; +import org.apache.skywalking.oap.log.analyzer.v2.provider.LogAnalyzerModuleConfig; +import org.apache.skywalking.oap.log.analyzer.v2.module.LogAnalyzerModule; +import org.apache.skywalking.oap.log.analyzer.v2.provider.LogAnalyzerModuleProvider; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.ConfigService; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; +import org.apache.skywalking.oap.server.core.query.type.ContentType; +import org.apache.skywalking.oap.server.core.source.Log; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class EnvoyAccessLogBuilderTest { + + private NamingControl namingControl; + + @BeforeEach + void setUp() { + namingControl = new NamingControl(512, 512, 512, new EndpointNameGrouping()); + } + + @Test + void toLogSerializesAccessLogEntryAsJsonContent() { + final EnvoyAccessLogBuilder builder = new EnvoyAccessLogBuilder(); + + final HTTPAccessLogEntry entry = HTTPAccessLogEntry.newBuilder() + .setResponse(HTTPResponseProperties.newBuilder() + .setResponseCode(UInt32Value.of(500))) + .build(); + + builder.setService("test-svc"); + builder.setTimestamp(1609459200000L); + + // init() receives the HTTPAccessLogEntry directly (matching inputType) + builder.init(entry, namingControl); + + final Log log = builder.toLog(); + + assertNotNull(log); + assertEquals(ContentType.JSON, log.getContentType()); + assertNotNull(log.getContent()); + // The JSON content should contain the response code from the access log entry + assertTrue(log.getContent().contains("500"), "Expected JSON content to contain '500' but got: " + log.getContent()); + } + + @Test + void toLogWithDefaultEntryHasNoContent() { + final EnvoyAccessLogBuilder builder = new EnvoyAccessLogBuilder(); + + builder.setService("test-svc"); + builder.setTimestamp(1609459200000L); + + // Pass a default (empty) entry — no response code, so toLog() serializes empty JSON + builder.init(HTTPAccessLogEntry.getDefaultInstance(), namingControl); + + final Log log = builder.toLog(); + + assertNotNull(log); + // Default entry still produces JSON (empty proto serializes to "{}") + // but content is set because accessLogEntry is non-null + assertNotNull(log.getContent()); + } + + @Test + void nameReturnsEnvoyAccessLog() { + assertEquals("EnvoyAccessLog", new EnvoyAccessLogBuilder().name()); + } + + /** + * Compiles and executes a LAL script that accesses both proto fields from + * the HTTPAccessLogEntry (via {@code parsed.*}) and entity fields from + * LogData (via {@code log.service}). Verifies at runtime that: + * <ul> + * <li>Proto fields are read correctly (response code extracted as tag)</li> + * <li>{@code log.service} is accessible and reads from LogData</li> + * <li>The output object is an {@link EnvoyAccessLogBuilder}</li> + * </ul> + */ + @Test + void executeLalWithProtoFieldAndEntityAccess() throws Exception { + final LALClassGenerator generator = new LALClassGenerator(); + generator.setInputType(HTTPAccessLogEntry.class); + generator.setOutputType(EnvoyAccessLogBuilder.class); + + // A simplified envoy-als script that: + // 1. Reads parsed?.response?.responseCode from the proto (tag extraction) + // 2. Reads log.service from LogData (tag extraction to verify entity access) + final String dsl = + "filter {\n" + + " extractor {\n" + + " tag 'status.code': parsed?.response?.responseCode?.value\n" + + " tag 'svc': log.service\n" + + " }\n" + + " sink {}\n" + + "}"; + + final LalExpression expr = generator.compile(dsl); + + // Build LogData with service/instance (simulates what LogsPersistence creates) + final LogData.Builder logData = LogData.newBuilder() + .setService("envoy-test-svc") + .setServiceInstance("envoy-test-instance") + .setTimestamp(1609459200000L) + .setLayer("MESH"); + + // Build HTTPAccessLogEntry with response code 503 + final HTTPAccessLogEntry entry = HTTPAccessLogEntry.newBuilder() + .setResponse(HTTPResponseProperties.newBuilder() + .setResponseCode(UInt32Value.of(503))) + .setCommonProperties(AccessLogCommon.newBuilder() + .setUpstreamCluster("outbound|80||backend.default.svc")) + .build(); + + // Execute + final FilterSpec filterSpec = buildFilterSpec(); + final ExecutionContext ctx = new ExecutionContext(); + ctx.log(logData); + ctx.extraLog(entry); + expr.execute(filterSpec, ctx); + + // Verify output is EnvoyAccessLogBuilder + assertNotNull(ctx.output()); + assertTrue(ctx.output() instanceof EnvoyAccessLogBuilder, + "Expected EnvoyAccessLogBuilder but got: " + ctx.output().getClass().getName()); + + // Verify proto field access and entity access via the output builder. + // Tags are stored in the output builder (via addTag), not in LogData. + // To verify, call init + toLog and check the Log's searchable tags. + final EnvoyAccessLogBuilder output = (EnvoyAccessLogBuilder) ctx.output(); + output.setSearchableTagKeys(java.util.Arrays.asList("status.code", "svc")); + // init() receives the HTTPAccessLogEntry directly (matching inputType) + output.init(entry, namingControl); + final Log log = output.toLog(); + + assertTrue(log.getTags().stream().anyMatch( + t -> "status.code".equals(t.getKey()) && "503".equals(t.getValue())), + "Expected tag status.code=503 from proto field, got: " + log.getTags()); + assertTrue(log.getTags().stream().anyMatch( + t -> "svc".equals(t.getKey()) && "envoy-test-svc".equals(t.getValue())), + "Expected tag svc=envoy-test-svc from log.service, got: " + log.getTags()); + } + + private FilterSpec buildFilterSpec() throws Exception { + final ModuleManager manager = mock(ModuleManager.class); + // Set isInPrepareStage = false via reflection + final java.lang.reflect.Field f = ModuleManager.class.getDeclaredField("isInPrepareStage"); + f.setAccessible(true); + f.set(manager, false); + + when(manager.find(anyString())) + .thenReturn(mock(ModuleProviderHolder.class)); + + final ModuleProviderHolder logHolder = mock(ModuleProviderHolder.class); + final LogAnalyzerModuleProvider logProvider = mock(LogAnalyzerModuleProvider.class); + when(logProvider.getMetricConverts()).thenReturn(Collections.emptyList()); + when(logHolder.provider()).thenReturn(logProvider); + when(manager.find(LogAnalyzerModule.NAME)).thenReturn(logHolder); + + final ModuleProviderHolder coreHolder = mock(ModuleProviderHolder.class); + final ModuleServiceHolder coreServices = mock(ModuleServiceHolder.class); + when(coreHolder.provider()).thenReturn(coreServices); + when(manager.find(CoreModule.NAME)).thenReturn(coreHolder); + + when(coreServices.getService(SourceReceiver.class)) + .thenReturn(mock(SourceReceiver.class)); + when(coreServices.getService(NamingControl.class)) + .thenReturn(new NamingControl(200, 200, 200, new EndpointNameGrouping())); + final ConfigService configService = mock(ConfigService.class); + when(configService.getSearchableLogsTags()).thenReturn(""); + when(coreServices.getService(ConfigService.class)).thenReturn(configService); + + final FilterSpec filterSpec = new FilterSpec(manager, new LogAnalyzerModuleConfig()); + // Empty sink listeners — we only test extractor behavior + final java.lang.reflect.Field slf = FilterSpec.class.getDeclaredField("sinkListenerFactories"); + slf.setAccessible(true); + slf.set(filterSpec, Collections.emptyList()); + + return filterSpec; + } + + private static void assertTrue(final boolean condition, final String message) { + org.junit.jupiter.api.Assertions.assertTrue(condition, message); + } +} diff --git a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java index 28afc17b1b..b2b526eef8 100644 --- a/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java +++ b/oap-server/server-receiver-plugin/otel-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/otel/otlp/OpenTelemetryLogHandler.java @@ -25,6 +25,7 @@ import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.logs.v1.LogRecord; +import java.util.Optional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.Getter; @@ -130,7 +131,7 @@ public class OpenTelemetryLogHandler .setTags(buildTags(logRecord)) .setBody(buildBody(logRecord)) .setLayer(layer), - null); + Optional.empty()); } catch (Exception e) { log.error("Failed to analyze logs", e); } diff --git a/oap-server/server-receiver-plugin/skywalking-log-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/log/provider/handler/grpc/LogReportServiceGrpcHandler.java b/oap-server/server-receiver-plugin/skywalking-log-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/log/provider/handler/grpc/LogReportServiceGrpcHandler.java index ad019c968b..d243142b60 100644 --- a/oap-server/server-receiver-plugin/skywalking-log-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/log/provider/handler/grpc/LogReportServiceGrpcHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-log-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/log/provider/handler/grpc/LogReportServiceGrpcHandler.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.receiver.log.provider.handler.grpc; import io.grpc.stub.StreamObserver; +import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.logging.v3.LogData; @@ -92,7 +93,7 @@ public class LogReportServiceGrpcHandler extends LogReportServiceGrpc.LogReportS try { LogData.Builder builder = logData.toBuilder(); setServiceName(builder); - logAnalyzerService.doAnalysis(builder, null); + logAnalyzerService.doAnalysis(builder, Optional.empty()); } catch (Exception e) { errorCounter.inc(); log.error(e.getMessage(), e); diff --git a/oap-server/server-receiver-plugin/skywalking-log-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/log/provider/handler/rest/LogReportServiceHTTPHandler.java b/oap-server/server-receiver-plugin/skywalking-log-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/log/provider/handler/rest/LogReportServiceHTTPHandler.java index 007b3d36b3..32a3a761fc 100644 --- a/oap-server/server-receiver-plugin/skywalking-log-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/log/provider/handler/rest/LogReportServiceHTTPHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-log-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/log/provider/handler/rest/LogReportServiceHTTPHandler.java @@ -19,6 +19,7 @@ package org.apache.skywalking.oap.server.receiver.log.provider.handler.rest; import com.linecorp.armeria.server.annotation.Post; import java.util.List; +import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.logging.v3.LogData; @@ -63,7 +64,7 @@ public class LogReportServiceHTTPHandler { @Post("/v3/logs") public Commands collectLogs(final List<LogData> logs) { try (final HistogramMetrics.Timer ignored = histogram.createTimer()) { - logs.forEach(it -> logAnalyzerService.doAnalysis(it, null)); + logs.forEach(it -> logAnalyzerService.doAnalysis(it, Optional.empty())); return Commands.newBuilder().build(); } catch (final Throwable e) { errorCounter.inc(); diff --git a/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/DSLClassGeneratorTest.java b/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/DSLClassGeneratorTest.java new file mode 100644 index 0000000000..13274bea2a --- /dev/null +++ b/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/DSLClassGeneratorTest.java @@ -0,0 +1,653 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.starter; + +import java.io.File; +import java.io.FileReader; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javassist.ClassPool; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.skywalking.oal.v2.generator.CodeGenModel; +import org.apache.skywalking.oal.v2.generator.MetricDefinitionEnricher; +import org.apache.skywalking.oal.v2.generator.OALClassGeneratorV2; +import org.apache.skywalking.oal.v2.model.MetricDefinition; +import org.apache.skywalking.oal.v2.parser.OALScriptParserV2; +import org.apache.skywalking.oap.log.analyzer.v2.compiler.LALClassGenerator; +import org.apache.skywalking.oap.log.analyzer.v2.spi.LALSourceTypeProvider; +import org.apache.skywalking.oap.meter.analyzer.v2.compiler.MALClassGenerator; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.SourceDecoratorManager; +import org.apache.skywalking.oap.server.core.config.v2.compiler.HierarchyRuleClassGenerator; +import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.source.LALOutputBuilder; +import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Compiles all DSL scripts (OAL, MAL, LAL, Hierarchy) from server-starter + * resources and dumps generated .class files to + * {@code target/generated-dsl-classes/} for inspection. + * + * <p>This does not start the OAP server or the module system. It only parses + * and compiles the DSL scripts, producing bytecode that can be decompiled + * with IDE tools or {@code javap}. + */ +@Slf4j +public class DSLClassGeneratorTest { + + private static final String SOURCE_PACKAGE = "org.apache.skywalking.oap.server.core.source."; + private static final String BROWSER_SOURCE_PACKAGE = "org.apache.skywalking.oap.server.core.browser.source."; + private static final String METRICS_PACKAGE = "org.apache.skywalking.oap.server.core.source.oal.rt.metrics."; + + private static final String[] POSSIBLE_PATHS = { + "oap-server/server-starter/src/main/resources/", + "../server-starter/src/main/resources/", + "src/main/resources/" + }; + + private static final String[] MAL_DIRS = { + "otel-rules", + "meter-analyzer-config", + "envoy-metrics-rules", + "log-mal-rules" + }; + + private static File RESOURCES_DIR; + private static File OUTPUT_BASE; + + @BeforeAll + public static void setup() { + RESOURCES_DIR = resolveResourcesDir(); + assertNotNull(RESOURCES_DIR, "Cannot find server-starter resources directory. " + + "Tried: " + String.join(", ", POSSIBLE_PATHS)); + + OUTPUT_BASE = new File("target/generated-dsl-classes"); + OUTPUT_BASE.mkdirs(); + + log.info("Resources: {}", RESOURCES_DIR.getAbsolutePath()); + log.info("Output: {}", OUTPUT_BASE.getAbsolutePath()); + } + + @Test + public void generateAllDSLClasses() { + final List<String> errors = new ArrayList<>(); + int oalClasses = 0; + int malExpressions = 0; + int malFilters = 0; + int lalExpressions = 0; + int hierarchyRules = 0; + + // ── OAL ── + log.info("--- OAL ---"); + initializeScopes(); + + final File oalDir = new File(OUTPUT_BASE, "oal"); + oalDir.mkdirs(); + OALClassGeneratorV2.setGeneratedFilePath(oalDir.getAbsolutePath()); + + final Map<String, OALDefine> defines = buildOALDefines(); + final ClassPool oalClassPool = new ClassPool(true); + + for (final Map.Entry<String, OALDefine> entry : defines.entrySet()) { + final String name = entry.getKey(); + final OALDefine define = entry.getValue(); + final File file = new File(RESOURCES_DIR, define.getConfigFile()); + if (!file.isFile()) { + errors.add("OAL: file not found: " + file); + continue; + } + try (FileReader reader = new FileReader(file)) { + final OALScriptParserV2 parser = + OALScriptParserV2.parse(reader, define.getConfigFile()); + final List<MetricDefinition> metrics = parser.getMetrics(); + final List<String> disabled = parser.getDisabledSources(); + + if (metrics.isEmpty()) { + log.info(" {}: 0 metrics, {} disabled sources", name, disabled.size()); + continue; + } + + final MetricDefinitionEnricher enricher = new MetricDefinitionEnricher( + define.getSourcePackage(), METRICS_PACKAGE + ); + final List<CodeGenModel> models = new ArrayList<>(); + for (final MetricDefinition m : metrics) { + models.add(enricher.enrich(m)); + } + + final OALClassGeneratorV2 gen = new OALClassGeneratorV2(define, oalClassPool); + gen.setOpenEngineDebug(true); + gen.setStorageBuilderFactory(new StorageBuilderFactory.Default()); + + final List<Class> metricsClasses = new ArrayList<>(); + final List<Class> dispatcherClasses = new ArrayList<>(); + gen.generateClassAtRuntime(models, disabled, metricsClasses, dispatcherClasses); + + final int count = metricsClasses.size() + dispatcherClasses.size(); + oalClasses += count; + log.info(" {}: {} metrics -> {} classes", name, metrics.size(), count); + } catch (Exception e) { + errors.add("OAL/" + name + ": " + e.getMessage()); + log.error(" {}: FAILED - {}", name, e.getMessage()); + } + } + + // ── MAL ── + log.info("--- MAL ---"); + final File malDir = new File(OUTPUT_BASE, "mal"); + malDir.mkdirs(); + + final MALClassGenerator malGen = new MALClassGenerator(); + malGen.setClassOutputDir(malDir); + + for (final String dirName : MAL_DIRS) { + final File dir = new File(RESOURCES_DIR, dirName); + if (!dir.isDirectory()) { + log.info(" {}: directory not found, skipping", dirName); + continue; + } + + for (final File yamlFile : findYamlFiles(dir)) { + final int[] counts = compileMALFile(malGen, dir, yamlFile, errors); + malExpressions += counts[0]; + malFilters += counts[1]; + } + } + log.info(" Total: {} expressions, {} filters", malExpressions, malFilters); + + // ── LAL ── + log.info("--- LAL ---"); + final File lalDir = new File(OUTPUT_BASE, "lal"); + lalDir.mkdirs(); + + lalExpressions = compileLAL(lalDir, errors); + log.info(" Total: {} expressions", lalExpressions); + + // ── Hierarchy ── + log.info("--- Hierarchy ---"); + final File hierarchyDir = new File(OUTPUT_BASE, "hierarchy"); + hierarchyDir.mkdirs(); + + hierarchyRules = compileHierarchy(hierarchyDir, errors); + log.info(" Total: {} rules", hierarchyRules); + + // ── Summary ── + log.info("=== Summary ==="); + log.info("OAL: {} classes", oalClasses); + log.info("MAL: {} expressions, {} filters", malExpressions, malFilters); + log.info("LAL: {} expressions", lalExpressions); + log.info("Hierarchy: {} rules", hierarchyRules); + log.info("Output: {}", OUTPUT_BASE.getAbsolutePath()); + + if (!errors.isEmpty()) { + log.error("=== {} Failures ===", errors.size()); + errors.forEach(e -> log.error(" - {}", e)); + fail("DSL compilation failures: " + errors.size() + "\n" + + String.join("\n", errors)); + } + + assertTrue(oalClasses > 100, "Expected > 100 OAL classes, got " + oalClasses); + assertTrue(malExpressions > 100, "Expected > 100 MAL expressions, got " + malExpressions); + assertTrue(lalExpressions > 0, "Expected > 0 LAL expressions, got " + lalExpressions); + assertTrue(hierarchyRules > 0, "Expected > 0 hierarchy rules, got " + hierarchyRules); + } + + // ──────────────────────────── MAL ──────────────────────────── + + @SuppressWarnings("unchecked") + private static int[] compileMALFile(final MALClassGenerator generator, + final File baseDir, + final File yamlFile, + final List<String> errors) { + int expressions = 0; + int filters = 0; + + try (Reader reader = new FileReader(yamlFile)) { + final Map<String, Object> config = new Yaml().load(reader); + if (config == null) { + return new int[]{0, 0}; + } + + final String relPath = baseDir.toPath().relativize(yamlFile.toPath()).toString(); + final String sourceName = relPath.substring(0, relPath.lastIndexOf('.')); + final String yamlSource = yamlFile.getName(); + + final String expPrefix = (String) config.get("expPrefix"); + final String expSuffix = (String) config.get("expSuffix"); + final String metricPrefix = (String) config.get("metricPrefix"); + final String filterText = (String) config.get("filter"); + + // Compile filter + if (filterText != null && !filterText.trim().isEmpty()) { + try { + generator.setClassNameHint("filter"); + generator.setYamlSource(yamlSource); + generator.compileFilter(filterText); + filters++; + } catch (Exception e) { + errors.add("MAL-filter/" + sourceName + ": " + e.getMessage()); + log.error(" {}: filter FAILED - {}", sourceName, e.getMessage()); + } finally { + generator.setClassNameHint(null); + generator.setYamlSource(null); + } + } + + // Compile expression rules + List<Map<String, String>> rules = + (List<Map<String, String>>) config.get("metricsRules"); + if (rules == null) { + rules = (List<Map<String, String>>) config.get("metrics"); + } + if (rules == null) { + return new int[]{expressions, filters}; + } + + for (int i = 0; i < rules.size(); i++) { + final Map<String, String> rule = rules.get(i); + final String ruleName = rule.get("name"); + final String exp = rule.get("exp"); + if (exp == null || exp.trim().isEmpty()) { + continue; + } + + final String fullExp = formatExp(expPrefix, expSuffix, exp); + final String metricName = metricPrefix != null + ? metricPrefix + "_" + ruleName : ruleName; + + try { + generator.setClassNameHint(ruleName); + generator.setYamlSource(yamlSource + ":" + i); + generator.compile(metricName, fullExp); + expressions++; + } catch (Exception e) { + errors.add("MAL/" + sourceName + "/" + ruleName + ": " + e.getMessage()); + log.error(" {}/{}: FAILED - {}", sourceName, ruleName, e.getMessage()); + } finally { + generator.setClassNameHint(null); + generator.setYamlSource(null); + } + } + } catch (Exception e) { + errors.add("MAL/" + yamlFile.getName() + ": " + e.getMessage()); + log.error(" {}: FAILED - {}", yamlFile.getName(), e.getMessage()); + } + + return new int[]{expressions, filters}; + } + + // ──────────────────────────── LAL ──────────────────────────── + + @SuppressWarnings("unchecked") + private static int compileLAL(final File lalDir, final List<String> errors) { + int count = 0; + + // SPI: inputType/outputType per layer + final Map<Layer, LALSourceTypeProvider> spiProviders = new HashMap<>(); + for (final LALSourceTypeProvider p : ServiceLoader.load(LALSourceTypeProvider.class)) { + spiProviders.put(p.layer(), p); + log.info(" SPI: layer={}, inputType={}", p.layer().name(), p.inputType().getName()); + } + + // SPI: LALOutputBuilder short names + final Map<String, Class<?>> outputBuilderNames = new HashMap<>(); + for (final LALOutputBuilder builder : ServiceLoader.load(LALOutputBuilder.class)) { + outputBuilderNames.put(builder.name(), builder.getClass()); + } + + final File lalResDir = new File(RESOURCES_DIR, "lal"); + if (!lalResDir.isDirectory()) { + log.info(" lal/ directory not found, skipping"); + return 0; + } + + final File[] yamlFiles = lalResDir.listFiles( + (dir, name) -> name.endsWith(".yaml") || name.endsWith(".yml") + ); + if (yamlFiles == null) { + return 0; + } + + for (final File yamlFile : yamlFiles) { + try (Reader reader = new FileReader(yamlFile)) { + final Map<String, Object> config = new Yaml().load(reader); + if (config == null) { + continue; + } + + final List<Map<String, Object>> rules = + (List<Map<String, Object>>) config.get("rules"); + if (rules == null) { + continue; + } + + for (final Map<String, Object> rule : rules) { + final String ruleName = (String) rule.get("name"); + final String dsl = (String) rule.get("dsl"); + final String layerStr = (String) rule.get("layer"); + final String inputTypeStr = (String) rule.get("inputType"); + final String outputTypeStr = (String) rule.get("outputType"); + + if (dsl == null || dsl.trim().isEmpty()) { + continue; + } + + try { + final LALClassGenerator gen = new LALClassGenerator(); + gen.setClassOutputDir(lalDir); + gen.setClassNameHint(ruleName); + gen.setYamlSource(yamlFile.getName()); + gen.setInputType(resolveInputType(inputTypeStr, layerStr, spiProviders)); + gen.setOutputType(resolveOutputType( + outputTypeStr, layerStr, spiProviders, outputBuilderNames)); + + gen.compile(dsl); + count++; + } catch (Exception e) { + errors.add("LAL/" + yamlFile.getName() + "/" + ruleName + + ": " + e.getMessage()); + log.error(" {}/{}: FAILED - {}", + yamlFile.getName(), ruleName, e.getMessage()); + } + } + } catch (Exception e) { + errors.add("LAL/" + yamlFile.getName() + ": " + e.getMessage()); + log.error(" {}: FAILED - {}", yamlFile.getName(), e.getMessage()); + } + } + return count; + } + + // ──────────────────────────── Hierarchy ──────────────────────────── + + @SuppressWarnings("unchecked") + private static int compileHierarchy(final File hierarchyDir, final List<String> errors) { + int count = 0; + final File hierarchyYml = new File(RESOURCES_DIR, "hierarchy-definition.yml"); + if (!hierarchyYml.isFile()) { + log.info(" hierarchy-definition.yml not found, skipping"); + return 0; + } + + try (Reader reader = new FileReader(hierarchyYml)) { + final Map<String, Map> config = new Yaml().loadAs(reader, Map.class); + final Map<String, String> ruleExpressions = + (Map<String, String>) config.get("auto-matching-rules"); + + if (ruleExpressions == null || ruleExpressions.isEmpty()) { + log.info(" No auto-matching-rules found"); + return 0; + } + + final HierarchyRuleClassGenerator gen = new HierarchyRuleClassGenerator(); + gen.setClassOutputDir(hierarchyDir); + gen.setYamlSource("hierarchy-definition.yml"); + + for (final Map.Entry<String, String> entry : ruleExpressions.entrySet()) { + final String ruleName = entry.getKey(); + try { + gen.setClassNameHint(ruleName); + gen.compile(ruleName, entry.getValue()); + count++; + } catch (Exception e) { + errors.add("Hierarchy/" + ruleName + ": " + e.getMessage()); + log.error(" {}: FAILED - {}", ruleName, e.getMessage()); + } + } + } catch (Exception e) { + errors.add("Hierarchy: " + e.getMessage()); + log.error(" FAILED - {}", e.getMessage()); + } + return count; + } + + // ──────────────────────────── Helpers ──────────────────────────── + + private static Map<String, OALDefine> buildOALDefines() { + final Map<String, OALDefine> defines = new LinkedHashMap<>(); + defines.put("core", oalDefine("oal/core.oal", SOURCE_PACKAGE, "")); + defines.put("java-agent", oalDefine("oal/java-agent.oal", SOURCE_PACKAGE, "")); + defines.put("dotnet-agent", oalDefine("oal/dotnet-agent.oal", SOURCE_PACKAGE, "")); + defines.put("browser", oalDefine("oal/browser.oal", BROWSER_SOURCE_PACKAGE, "")); + defines.put("mesh", oalDefine("oal/mesh.oal", SOURCE_PACKAGE, "ServiceMesh")); + defines.put("tcp", oalDefine("oal/tcp.oal", SOURCE_PACKAGE, "EnvoyTCP")); + defines.put("ebpf", oalDefine("oal/ebpf.oal", SOURCE_PACKAGE, "")); + defines.put("cilium", oalDefine("oal/cilium.oal", SOURCE_PACKAGE, "")); + defines.put("disable", oalDefine("oal/disable.oal", SOURCE_PACKAGE, "")); + return defines; + } + + private static OALDefine oalDefine(final String configFile, + final String sourcePackage, + final String catalog) { + return new OALDefine(configFile, sourcePackage, catalog) { + }; + } + + private static File resolveResourcesDir() { + for (final String path : POSSIBLE_PATHS) { + final File dir = new File(path); + if (dir.isDirectory() && new File(dir, "oal").isDirectory()) { + return dir; + } + } + return null; + } + + private static List<File> findYamlFiles(final File dir) { + try (Stream<Path> stream = Files.walk(dir.toPath())) { + return stream + .filter(Files::isRegularFile) + .filter(p -> { + final String name = p.getFileName().toString(); + return (name.endsWith(".yaml") || name.endsWith(".yml")) + && !name.endsWith(".data.yaml"); + }) + .map(Path::toFile) + .sorted() + .collect(Collectors.toList()); + } catch (Exception e) { + log.warn("Failed to walk directory: {}", dir, e); + return new ArrayList<>(); + } + } + + /** + * Replicates {@code MetricConvert.formatExp()}. + */ + private static String formatExp(final String expPrefix, + final String expSuffix, + final String exp) { + String ret = exp; + if (expPrefix != null && !expPrefix.isEmpty()) { + ret = String.format("(%s.%s)", StringUtils.substringBefore(exp, "."), expPrefix); + final String after = StringUtils.substringAfter(exp, "."); + if (after != null && !after.isEmpty()) { + ret = ret + "." + after; + } + } + if (expSuffix != null && !expSuffix.isEmpty()) { + final int insertIdx = ret.indexOf('.'); + if (insertIdx > 0) { + ret = ret.substring(0, insertIdx + 1) + expSuffix + "." + + ret.substring(insertIdx + 1); + } + } + return ret; + } + + private static Class<?> resolveInputType( + final String yamlType, + final String layerStr, + final Map<Layer, LALSourceTypeProvider> spiProviders) { + if (yamlType != null && !yamlType.isEmpty()) { + try { + return Class.forName(yamlType); + } catch (ClassNotFoundException e) { + log.warn("inputType class not found: {}", yamlType); + } + } + if (layerStr != null) { + final Layer layer = Layer.nameOf(layerStr); + final LALSourceTypeProvider spi = spiProviders.get(layer); + if (spi != null) { + return spi.inputType(); + } + } + return null; + } + + private static Class<?> resolveOutputType( + final String yamlType, + final String layerStr, + final Map<Layer, LALSourceTypeProvider> spiProviders, + final Map<String, Class<?>> outputBuilderNames) { + if (yamlType != null && !yamlType.isEmpty()) { + if (!yamlType.contains(".")) { + final Class<?> byName = outputBuilderNames.get(yamlType); + if (byName != null) { + return byName; + } + } + try { + return Class.forName(yamlType); + } catch (ClassNotFoundException e) { + log.warn("outputType class not found: {}", yamlType); + } + } + if (layerStr != null) { + final Layer layer = Layer.nameOf(layerStr); + final LALSourceTypeProvider spi = spiProviders.get(layer); + if (spi != null) { + return spi.outputType(); + } + } + return null; + } + + /** + * Register all OAL source scopes and decorators. + */ + private static void initializeScopes() { + final DefaultScopeDefine.Listener listener = new DefaultScopeDefine.Listener(); + + notifyClass(listener, SOURCE_PACKAGE, "Service"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstance"); + notifyClass(listener, SOURCE_PACKAGE, "Endpoint"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "EndpointRelation"); + notifyClass(listener, SOURCE_PACKAGE, "DatabaseAccess"); + notifyClass(listener, SOURCE_PACKAGE, "CacheAccess"); + notifyClass(listener, SOURCE_PACKAGE, "MQAccess"); + notifyClass(listener, SOURCE_PACKAGE, "MQEndpointAccess"); + + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMCPU"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMMemory"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMMemoryPool"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMGC"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMThread"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMClass"); + + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceCLRCPU"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceCLRGC"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceCLRThread"); + + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppTraffic"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppPageTraffic"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppSingleVersionTraffic"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppPerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppPagePerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppSingleVersionPerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppResourcePerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppWebInteractionPerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppWebVitalsPerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserErrorLog"); + + notifyClass(listener, SOURCE_PACKAGE, "ServiceMesh"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceMeshService"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceMeshServiceInstance"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceMeshServiceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceMeshServiceInstanceRelation"); + + notifyClass(listener, SOURCE_PACKAGE, "TCPService"); + notifyClass(listener, SOURCE_PACKAGE, "TCPServiceInstance"); + notifyClass(listener, SOURCE_PACKAGE, "TCPServiceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "TCPServiceInstanceRelation"); + + notifyClass(listener, SOURCE_PACKAGE, "EBPFProfilingSchedule"); + + notifyClass(listener, SOURCE_PACKAGE, "CiliumService"); + notifyClass(listener, SOURCE_PACKAGE, "CiliumServiceInstance"); + notifyClass(listener, SOURCE_PACKAGE, "CiliumEndpoint"); + notifyClass(listener, SOURCE_PACKAGE, "CiliumServiceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "CiliumServiceInstanceRelation"); + + notifyClass(listener, SOURCE_PACKAGE, "K8SService"); + notifyClass(listener, SOURCE_PACKAGE, "K8SServiceInstance"); + notifyClass(listener, SOURCE_PACKAGE, "K8SEndpoint"); + notifyClass(listener, SOURCE_PACKAGE, "K8SServiceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "K8SServiceInstanceRelation"); + + notifyClass(listener, SOURCE_PACKAGE, "Process"); + notifyClass(listener, SOURCE_PACKAGE, "ProcessRelation"); + + registerDecorator(SOURCE_PACKAGE, "ServiceDecorator"); + registerDecorator(SOURCE_PACKAGE, "EndpointDecorator"); + registerDecorator(SOURCE_PACKAGE, "K8SServiceDecorator"); + registerDecorator(SOURCE_PACKAGE, "K8SEndpointDecorator"); + } + + private static void notifyClass(final DefaultScopeDefine.Listener listener, + final String packageName, + final String className) { + try { + listener.notify(Class.forName(packageName + className)); + } catch (Exception e) { + log.debug("Scope {} registration: {}", className, e.getMessage()); + } + } + + private static void registerDecorator(final String packageName, + final String decoratorName) { + try { + final SourceDecoratorManager manager = new SourceDecoratorManager(); + manager.addIfAsSourceDecorator(Class.forName(packageName + decoratorName)); + } catch (Exception e) { + log.debug("Decorator {} registration: {}", decoratorName, e.getMessage()); + } + } +}
