wu-sheng commented on code in PR #13699: URL: https://github.com/apache/skywalking/pull/13699#discussion_r2794098817
########## oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/parser/RealOALScriptsTest.java: ########## @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oal.v2.parser; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oal.v2.model.MetricDefinition; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Test V2 parser with real OAL scripts from server-starter/resources/oal. + * + * This validates that V2 can parse all production OAL scripts. + */ +@Slf4j +public class RealOALScriptsTest { + + /** + * Find the OAL scripts directory in the project. + * + * Tries multiple paths to locate the directory: + * 1. From current working directory (Maven default) + * 2. From user.dir system property + * 3. Relative from this module + */ + private File findOALScriptsDir() { + String[] possiblePaths = { + "oap-server/server-starter/src/main/resources/oal", + "../server-starter/src/main/resources/oal", + "../../server-starter/src/main/resources/oal" + }; + + for (String path : possiblePaths) { + File dir = new File(path); + if (dir.exists() && dir.isDirectory()) { + log.debug("Found OAL scripts directory at: {}", dir.getAbsolutePath()); + return dir; + } + } + + throw new IllegalStateException("Could not find OAL scripts directory. Tried: " + + String.join(", ", possiblePaths)); + } + + /** + * Test parsing core.oal with V2. + * + * core.oal contains the main service and endpoint metrics. + */ + @Test + public void testParseCoreOAL() throws IOException { + File oalDir = findOALScriptsDir(); + File oalFile = new File(oalDir, "core.oal"); + assertTrue(oalFile.exists(), "core.oal not found at: " + oalFile.getAbsolutePath()); + + OALScriptParserV2 parser = OALScriptParserV2.parse(new FileReader(oalFile), "core.oal"); + Review Comment: Fixed. All FileReader instances now use try-with-resources. ########## oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/RuntimeOALGenerationTest.java: ########## @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oal.v2.generator; + +import java.io.File; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javassist.ClassPool; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oal.v2.model.MetricDefinition; +import org.apache.skywalking.oal.v2.parser.OALScriptParserV2; +import org.apache.skywalking.oap.server.core.analysis.SourceDecoratorManager; +import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Test OAL V2 generation with all runtime OAL scripts. + * + * <p>This test loads all OAL scripts defined via OALDefine implementations and generates + * classes exactly as the runtime does. Generated classes are written to target/generated-test-sources + * for IDE inspection. + * + * <p>Key behaviors: + * <ul> + * <li>Scans all OALDefine implementations to map OAL scripts</li> + * <li>Verifies OAL scripts exist and can be parsed</li> + * <li>Detects dispatcher conflicts when same source appears in multiple OAL files</li> + * <li>Generates classes with correct package structure matching runtime</li> + * <li>Allows developers to inspect generated sources via IDE decompiler</li> + * </ul> + */ +@Slf4j +public class RuntimeOALGenerationTest { + + private static final String SOURCE_PACKAGE = "org.apache.skywalking.oap.server.core.source."; + private static final String BROWSER_SOURCE_PACKAGE = "org.apache.skywalking.oap.server.core.browser.source."; + private static final String METRICS_PACKAGE = "org.apache.skywalking.oap.server.core.source.oal.rt.metrics."; + + private static final String[] POSSIBLE_PATHS = { + "oap-server/server-starter/src/main/resources/", + "../server-starter/src/main/resources/", + "../../server-starter/src/main/resources/" + }; + + /** + * All known OALDefine implementations mapped to their OAL script paths. + */ + private static final Map<String, OALDefine> OAL_DEFINES = new HashMap<>(); + + @BeforeAll + public static void setup() { + // Initialize all scopes + initializeScopes(); + + // Register all known OALDefine implementations (matching runtime OALDefine classes) + // CoreOALDefine - no catalog + registerOALDefine("core", createOALDefine("oal/core.oal", SOURCE_PACKAGE, "")); + // JVMOALDefine - no catalog + registerOALDefine("java-agent", createOALDefine("oal/java-agent.oal", SOURCE_PACKAGE, "")); + // CLROALDefine - no catalog + registerOALDefine("dotnet-agent", createOALDefine("oal/dotnet-agent.oal", SOURCE_PACKAGE, "")); + // BrowserOALDefine - different source package, no catalog + registerOALDefine("browser", createOALDefine("oal/browser.oal", BROWSER_SOURCE_PACKAGE, "")); + // MeshOALDefine - catalog: "ServiceMesh" + registerOALDefine("mesh", createOALDefine("oal/mesh.oal", SOURCE_PACKAGE, "ServiceMesh")); + // TCPOALDefine - catalog: "EnvoyTCP" + registerOALDefine("tcp", createOALDefine("oal/tcp.oal", SOURCE_PACKAGE, "EnvoyTCP")); + // EBPFOALDefine - no catalog + registerOALDefine("ebpf", createOALDefine("oal/ebpf.oal", SOURCE_PACKAGE, "")); + // CiliumOALDefine - no catalog + registerOALDefine("cilium", createOALDefine("oal/cilium.oal", SOURCE_PACKAGE, "")); + // DisableOALDefine - no catalog + registerOALDefine("disable", createOALDefine("oal/disable.oal", SOURCE_PACKAGE, "")); + + // Set generated file path for IDE inspection + OALClassGeneratorV2.setGeneratedFilePath("target/test-classes"); + } + + private static void registerOALDefine(String name, OALDefine define) { + OAL_DEFINES.put(name, define); + } + + private static OALDefine createOALDefine(String configFile, String sourcePackage, String catalog) { + return new OALDefine(configFile, sourcePackage, catalog) { + }; + } + + private static void initializeScopes() { + DefaultScopeDefine.Listener listener = new DefaultScopeDefine.Listener(); + + // Core sources + notifyClass(listener, SOURCE_PACKAGE, "Service"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstance"); + notifyClass(listener, SOURCE_PACKAGE, "Endpoint"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "EndpointRelation"); + notifyClass(listener, SOURCE_PACKAGE, "DatabaseAccess"); + notifyClass(listener, SOURCE_PACKAGE, "CacheAccess"); + notifyClass(listener, SOURCE_PACKAGE, "MQAccess"); + notifyClass(listener, SOURCE_PACKAGE, "MQEndpointAccess"); + + // JVM sources + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMCPU"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMMemory"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMMemoryPool"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMGC"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMThread"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceJVMClass"); + + // CLR sources + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceCLRCPU"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceCLRGC"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceInstanceCLRThread"); + + // Browser sources + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppTraffic"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppPageTraffic"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppSingleVersionTraffic"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppPerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppPagePerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppSingleVersionPerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppResourcePerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppWebInteractionPerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserAppWebVitalsPerf"); + notifyClass(listener, BROWSER_SOURCE_PACKAGE, "BrowserErrorLog"); + + // Mesh sources + notifyClass(listener, SOURCE_PACKAGE, "ServiceMesh"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceMeshService"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceMeshServiceInstance"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceMeshServiceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "ServiceMeshServiceInstanceRelation"); + + // TCP sources + notifyClass(listener, SOURCE_PACKAGE, "TCPService"); + notifyClass(listener, SOURCE_PACKAGE, "TCPServiceInstance"); + notifyClass(listener, SOURCE_PACKAGE, "TCPServiceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "TCPServiceInstanceRelation"); + + // eBPF sources + notifyClass(listener, SOURCE_PACKAGE, "EBPFProfilingSchedule"); + + // Cilium sources + notifyClass(listener, SOURCE_PACKAGE, "CiliumService"); + notifyClass(listener, SOURCE_PACKAGE, "CiliumServiceInstance"); + notifyClass(listener, SOURCE_PACKAGE, "CiliumEndpoint"); + notifyClass(listener, SOURCE_PACKAGE, "CiliumServiceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "CiliumServiceInstanceRelation"); + + // K8s sources + notifyClass(listener, SOURCE_PACKAGE, "K8SService"); + notifyClass(listener, SOURCE_PACKAGE, "K8SServiceInstance"); + notifyClass(listener, SOURCE_PACKAGE, "K8SEndpoint"); + notifyClass(listener, SOURCE_PACKAGE, "K8SServiceRelation"); + notifyClass(listener, SOURCE_PACKAGE, "K8SServiceInstanceRelation"); + + // Process sources + notifyClass(listener, SOURCE_PACKAGE, "Process"); + notifyClass(listener, SOURCE_PACKAGE, "ProcessRelation"); + + // Register decorators + registerDecorator(SOURCE_PACKAGE, "ServiceDecorator"); + registerDecorator(SOURCE_PACKAGE, "EndpointDecorator"); + registerDecorator(SOURCE_PACKAGE, "K8SServiceDecorator"); + registerDecorator(SOURCE_PACKAGE, "K8SEndpointDecorator"); + } + + private static void notifyClass(DefaultScopeDefine.Listener listener, String packageName, String className) { + try { + Class<?> clazz = Class.forName(packageName + className); + listener.notify(clazz); + } catch (Exception e) { + log.debug("Scope {} registration: {}", className, e.getMessage()); + } + } + + private static void registerDecorator(String packageName, String decoratorName) { + try { + Class<?> clazz = Class.forName(packageName + decoratorName); + SourceDecoratorManager manager = new SourceDecoratorManager(); + manager.addIfAsSourceDecorator(clazz); + } catch (Exception e) { + log.debug("Decorator {} registration: {}", decoratorName, e.getMessage()); + } + } + + /** + * Test that all OAL scripts can be loaded, parsed, and classes generated. + * + * <p>This test validates: + * <ul> + * <li>All OAL script files exist</li> + * <li>All scripts parse successfully</li> + * <li>Classes can be generated without conflicts</li> + * <li>Generated classes match runtime structure</li> + * </ul> + */ + @Test + public void testGenerateAllRuntimeOALScripts() throws Exception { + // Single ClassPool for all generated classes + // No conflicts because catalog prefix creates unique dispatcher class names + // (e.g., ServiceDispatcher vs ServiceMeshServiceDispatcher) + ClassPool classPool = new ClassPool(true); + + int totalMetrics = 0; + int totalDispatchers = 0; + int totalGeneratedClasses = 0; + List<String> errors = new ArrayList<>(); + + for (Map.Entry<String, OALDefine> entry : OAL_DEFINES.entrySet()) { + String oalName = entry.getKey(); + OALDefine define = entry.getValue(); + + File oalFile = findOALFile(define.getConfigFile()); + assertNotNull(oalFile, "OAL file not found: " + define.getConfigFile() + + ". Tried paths: " + String.join(", ", POSSIBLE_PATHS)); + + try { + // Parse OAL script + OALScriptParserV2 parser = OALScriptParserV2.parse(new FileReader(oalFile), define.getConfigFile()); + List<MetricDefinition> metrics = parser.getMetrics(); Review Comment: Fixed. Using try-with-resources now. ########## oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/v2/OALEngineV2.java: ########## @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oal.v2; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oal.v2.generator.CodeGenModel; +import org.apache.skywalking.oal.v2.generator.MetricDefinitionEnricher; +import org.apache.skywalking.oal.v2.generator.OALClassGeneratorV2; +import org.apache.skywalking.oal.v2.model.MetricDefinition; +import org.apache.skywalking.oal.v2.parser.OALScriptParserV2; +import org.apache.skywalking.oap.server.core.analysis.DispatcherDetectorListener; +import org.apache.skywalking.oap.server.core.analysis.StreamAnnotationListener; +import org.apache.skywalking.oap.server.core.oal.rt.OALCompileException; +import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; +import org.apache.skywalking.oap.server.core.oal.rt.OALEngine; +import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory; +import org.apache.skywalking.oap.server.core.storage.StorageException; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.util.ResourceUtils; + +/** + * V2 OAL Engine - completely independent implementation. + * + * This engine: + * 1. Parses OAL scripts using V2 parser (immutable models) + * 2. Enriches V2 models with metadata (MetricDefinitionEnricher) + * 3. Generates classes using V2 templates (OALClassGeneratorV2) + * + * Benefits: + * - Clean immutable data models + * - Type-safe filter values and function arguments + * - Better error messages with source location tracking + * - Completely independent (no V1 code dependencies) + */ +@Slf4j +public class OALEngineV2 implements OALEngine { + + private final OALClassGeneratorV2 classGeneratorV2; + private final OALDefine oalDefine; + + private StreamAnnotationListener streamAnnotationListener; + private DispatcherDetectorListener dispatcherDetectorListener; + private final List<Class> metricsClasses; + private final List<Class> dispatcherClasses; + + public OALEngineV2(OALDefine define) { + this.oalDefine = define; + this.classGeneratorV2 = new OALClassGeneratorV2(define); + this.metricsClasses = new ArrayList<>(); + this.dispatcherClasses = new ArrayList<>(); + } + + @Override + public void setStreamListener(StreamAnnotationListener listener) { + this.streamAnnotationListener = listener; + } + + @Override + public void setDispatcherListener(DispatcherDetectorListener listener) { + this.dispatcherDetectorListener = listener; + } + + @Override + public void setStorageBuilderFactory(StorageBuilderFactory factory) { + classGeneratorV2.setStorageBuilderFactory(factory); + } + + @Override + public void start(ClassLoader currentClassLoader) throws ModuleStartException, OALCompileException { + log.info("Starting OAL Engine V2..."); + + // Prepare temp folder for generated classes + classGeneratorV2.prepareRTTempFolder(); + classGeneratorV2.setCurrentClassLoader(currentClassLoader); + + // Load OAL script + Reader reader; + try { + reader = ResourceUtils.read(oalDefine.getConfigFile()); + } catch (FileNotFoundException e) { + throw new ModuleStartException("Can't locate " + oalDefine.getConfigFile(), e); + } + + // Parse using V2 parser + OALScriptParserV2 v2Parser; + try { + v2Parser = OALScriptParserV2.parse(reader, oalDefine.getConfigFile()); + log.info("V2 Parser: Successfully parsed {} metrics", v2Parser.getMetricsCount()); + } catch (IOException e) { + throw new ModuleStartException("OAL V2 script parse failure", e); + } + + // Enrich V2 models with metadata for code generation + List<CodeGenModel> codeGenModels = enrichMetrics(v2Parser.getMetrics()); + log.info("V2 Enricher: Enriched {} metrics with metadata", codeGenModels.size()); + + // Generate classes using V2 generator + classGeneratorV2.generateClassAtRuntime( + codeGenModels, + v2Parser.getDisabledSources(), + metricsClasses, + dispatcherClasses + ); + + log.info("OAL Engine V2 started successfully. Generated {} metrics classes, {} dispatcher classes", + metricsClasses.size(), + dispatcherClasses.size() + ); Review Comment: Fixed. Reader now wrapped in try-with-resources. ########## oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/v2/registry/MetricsFunctionRegistry.java: ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oal.v2.registry; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Registry for metrics aggregation functions. + * + * This interface abstracts the discovery and lookup of metrics functions + * annotated with @MetricsFunction. Implementations can use different strategies: + * - Classpath scanning (default) + * - Manual registration + * - Configuration-based registration + * + * Example usage: + * <pre> + * MetricsFunctionRegistry registry = MetricsFunctionRegistryFactory.createDefault(); + * + * Optional<MetricsFunctionDescriptor> longAvg = registry.findFunction("longAvg"); + * if (longAvg.isPresent()) { + * Class<? extends Metrics> metricsClass = longAvg.get().getMetricsClass(); + * Method entranceMethod = longAvg.get().getEntranceMethod(); + * // ... use for code generation + * } + * </pre> Review Comment: Fixed. Updated to reference DefaultMetricsFunctionRegistry.create(). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
