This is an automated email from the ASF dual-hosted git repository. wusheng pushed a commit to branch oal-v2 in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit c97a743bb511296ca8640cae50665fbe90747521 Author: Wu Sheng <[email protected]> AuthorDate: Tue Feb 10 13:05:31 2026 +0800 Add source generation and bytecode consistency verification - Add OALSourceGenerator to generate compilable Java source files using the same FreeMarker templates as bytecode generator - Add OALSourceGenerationTest to write sources to target/generated-test-sources/oal/ for debugging and documentation - Add SourceBytecodeConsistencyTest to verify 100% consistency between generated source files and Javassist bytecode loaded into JVM: - Template output consistency for all method bodies - Field declarations and type matching - Class structure (inheritance, interfaces) - All annotations (@Stream, @Column, @BanyanDB.*, @ElasticSearch.*) Co-Authored-By: Claude Opus 4.5 <[email protected]> --- .../oal/v2/generator/OALSourceGenerator.java | 310 +++++++++++++++++ .../oal/v2/generator/OALSourceGenerationTest.java | 386 ++++++++++++++++++++ .../generator/SourceBytecodeConsistencyTest.java | 387 +++++++++++++++++++++ 3 files changed, 1083 insertions(+) diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/v2/generator/OALSourceGenerator.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/v2/generator/OALSourceGenerator.java new file mode 100644 index 0000000000..08e8babc5a --- /dev/null +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/v2/generator/OALSourceGenerator.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oal.v2.generator; + +import freemarker.template.Configuration; +import freemarker.template.Version; +import java.io.StringWriter; +import java.util.Locale; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.oal.rt.OALCompileException; +import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; +import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory; + +/** + * Generates Java source code that exactly matches the bytecode produced by {@link OALClassGeneratorV2}. + * + * <p>This generator produces complete, compilable Java source files using the same FreeMarker + * templates used for bytecode generation. The generated source files are useful for: + * <ul> + * <li>Debugging and inspection of generated code</li> + * <li>Documentation of the code generation process</li> + * <li>Verification that templates produce correct Java syntax</li> + * </ul> + * + * <p>The generated sources are 100% consistent with the bytecode loaded into JVM because + * they use the identical FreeMarker templates for method body generation. + * + * @see OALClassGeneratorV2 + */ +@Slf4j +public class OALSourceGenerator { + + private static final String METRICS_FUNCTION_PACKAGE = "org.apache.skywalking.oap.server.core.analysis.metrics."; + private static final String METRICS_STREAM_PROCESSOR = "org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor"; + private static final String[] METRICS_CLASS_METHODS = { + "id", "hashCode", "remoteHashCode", "equals", "serialize", "deserialize", "getMeta", "toHour", "toDay" + }; + private static final String[] METRICS_BUILDER_CLASS_METHODS = { + "entity2Storage", "storage2Entity" + }; + + private final OALDefine oalDefine; + private final Configuration configuration; + private StorageBuilderFactory storageBuilderFactory; + + public OALSourceGenerator(OALDefine define) { + this.oalDefine = define; + this.configuration = new Configuration(new Version("2.3.28")); + this.configuration.setEncoding(Locale.ENGLISH, "UTF-8"); + this.configuration.setClassLoaderForTemplateLoading( + OALSourceGenerator.class.getClassLoader(), "/code-templates-v2"); + } + + public void setStorageBuilderFactory(StorageBuilderFactory factory) { + this.storageBuilderFactory = factory; + } + + /** + * Generate complete Java source code for a metrics class. + * + * <p>The generated source includes: + * <ul> + * <li>Package declaration</li> + * <li>Import statements</li> + * <li>Class-level @Stream annotation</li> + * <li>Class declaration extending the metrics function class</li> + * <li>Fields with @Column, @BanyanDB, @ElasticSearch annotations</li> + * <li>Getter/setter methods</li> + * <li>All template-generated methods (id, hashCode, equals, serialize, etc.)</li> + * </ul> + * + * @param model the code generation model + * @return complete Java source code as a string + * @throws OALCompileException if template processing fails + */ + public String generateMetricsSource(CodeGenModel model) throws OALCompileException { + StringBuilder source = new StringBuilder(); + String className = model.getMetricsName() + "Metrics"; + + // Package declaration (remove trailing dot if present) + String metricsPackage = oalDefine.getDynamicMetricsClassPackage(); + if (metricsPackage.endsWith(".")) { + metricsPackage = metricsPackage.substring(0, metricsPackage.length() - 1); + } + source.append("package ").append(metricsPackage).append(";\n\n"); + + // Imports - match what Javassist would require + source.append("import org.apache.skywalking.oap.server.core.analysis.Stream;\n"); + source.append("import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;\n"); + source.append("import org.apache.skywalking.oap.server.core.analysis.metrics.MetricsMetaInfo;\n"); + source.append("import org.apache.skywalking.oap.server.core.analysis.metrics.WithMetadata;\n"); + source.append("import org.apache.skywalking.oap.server.core.analysis.metrics.") + .append(model.getMetricsClassName()).append(";\n"); + source.append("import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;\n"); + source.append("import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;\n"); + source.append("import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;\n"); + source.append("import org.apache.skywalking.oap.server.core.storage.annotation.Column;\n"); + source.append("import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch;\n"); + source.append("\n"); + + // Class-level @Stream annotation - exactly as Javassist adds it + source.append("@Stream(\n"); + source.append(" name = \"").append(model.getTableName()).append("\",\n"); + source.append(" scopeId = ").append(model.getSourceScopeId()).append(",\n"); + source.append(" builder = ").append(model.getMetricsName()).append("MetricsBuilder.class,\n"); + source.append(" processor = MetricsStreamProcessor.class\n"); + source.append(")\n"); + + // Class declaration + source.append("public class ").append(className) + .append(" extends ").append(model.getMetricsClassName()) + .append(" implements WithMetadata {\n\n"); + + // Fields with annotations - exactly as Javassist adds them + for (CodeGenModel.SourceFieldV2 field : model.getFieldsFromSource()) { + // @Column annotation + source.append(" @Column(name = \"").append(field.getColumnName()).append("\""); + if (field.getType().equals(String.class)) { + source.append(", length = ").append(field.getLength()); + } + source.append(")\n"); + + // @BanyanDB.SeriesID and @ElasticSearch.EnableDocValues for ID fields + if (field.isID()) { + source.append(" @BanyanDB.SeriesID(index = 0)\n"); + source.append(" @ElasticSearch.EnableDocValues\n"); + } + + // @BanyanDB.ShardingKey for sharding key fields + if (field.isShardingKey()) { + source.append(" @BanyanDB.ShardingKey(index = ").append(field.getShardingKeyIdx()).append(")\n"); + } + + // Field declaration + source.append(" private ").append(field.getType().getName()) + .append(" ").append(field.getFieldName()).append(";\n\n"); + } + + // Default constructor + source.append(" public ").append(className).append("() {\n"); + source.append(" }\n\n"); + + // Getter/setter methods for each field + for (CodeGenModel.SourceFieldV2 field : model.getFieldsFromSource()) { + String capFieldName = field.getFieldName().substring(0, 1).toUpperCase() + + field.getFieldName().substring(1); + + // Getter + source.append(" public ").append(field.getType().getName()) + .append(" get").append(capFieldName).append("() {\n"); + source.append(" return this.").append(field.getFieldName()).append(";\n"); + source.append(" }\n\n"); + + // Setter + source.append(" public void set").append(capFieldName) + .append("(").append(field.getType().getName()).append(" ").append(field.getFieldName()).append(") {\n"); + source.append(" this.").append(field.getFieldName()).append(" = ").append(field.getFieldName()).append(";\n"); + source.append(" }\n\n"); + } + + // Template-generated methods - exactly what Javassist compiles + for (String method : METRICS_CLASS_METHODS) { + StringWriter methodEntity = new StringWriter(); + try { + configuration.getTemplate("metrics/" + method + ".ftl").process(model, methodEntity); + source.append(" ").append(methodEntity.toString().trim()).append("\n\n"); + } catch (Exception e) { + throw new OALCompileException("Failed to generate method " + method + ": " + e.getMessage(), e); + } + } + + source.append("}\n"); + return source.toString(); + } + + /** + * Generate complete Java source code for a metrics builder class. + * + * @param model the code generation model + * @return complete Java source code as a string + * @throws OALCompileException if template processing fails + */ + public String generateMetricsBuilderSource(CodeGenModel model) throws OALCompileException { + if (storageBuilderFactory == null) { + storageBuilderFactory = new StorageBuilderFactory.Default(); + } + + StringBuilder source = new StringBuilder(); + String className = model.getMetricsName() + "MetricsBuilder"; + + // Package declaration (remove trailing dot if present) + String builderPackage = oalDefine.getDynamicMetricsBuilderClassPackage(); + if (builderPackage.endsWith(".")) { + builderPackage = builderPackage.substring(0, builderPackage.length() - 1); + } + source.append("package ").append(builderPackage).append(";\n\n"); + + // Imports + source.append("import org.apache.skywalking.oap.server.core.storage.StorageData;\n"); + source.append("import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;\n"); + source.append("import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;\n"); + source.append("import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;\n"); + String metricsClassFqn = oalDefine.getDynamicMetricsClassPackage() + model.getMetricsName() + "Metrics"; + source.append("import ").append(metricsClassFqn).append(";\n"); + source.append("\n"); + + // Class declaration + source.append("public class ").append(className) + .append(" implements StorageBuilder {\n\n"); + + // Default constructor + source.append(" public ").append(className).append("() {\n"); + source.append(" }\n\n"); + + // Template-generated methods + for (String method : METRICS_BUILDER_CLASS_METHODS) { + StringWriter methodEntity = new StringWriter(); + try { + configuration.getTemplate( + storageBuilderFactory.builderTemplate().getTemplatePath() + "/" + method + ".ftl") + .process(model, methodEntity); + source.append(" ").append(methodEntity.toString().trim()).append("\n\n"); + } catch (Exception e) { + throw new OALCompileException("Failed to generate method " + method + ": " + e.getMessage(), e); + } + } + + source.append("}\n"); + return source.toString(); + } + + /** + * Generate complete Java source code for a dispatcher class. + * + * @param dispatcherContext the dispatcher context with all metrics + * @return complete Java source code as a string + * @throws OALCompileException if template processing fails + */ + public String generateDispatcherSource(OALClassGeneratorV2.DispatcherContextV2 dispatcherContext) + throws OALCompileException { + + StringBuilder source = new StringBuilder(); + String className = dispatcherContext.getSourceName() + "Dispatcher"; + + // Package declaration (remove trailing dot if present) + String dispatcherPackage = oalDefine.getDynamicDispatcherClassPackage(); + if (dispatcherPackage.endsWith(".")) { + dispatcherPackage = dispatcherPackage.substring(0, dispatcherPackage.length() - 1); + } + source.append("package ").append(dispatcherPackage).append(";\n\n"); + + // Imports + source.append("import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher;\n"); + source.append("import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;\n"); + source.append("import org.apache.skywalking.oap.server.core.source.ISource;\n"); + String sourceClassFqn = oalDefine.getSourcePackage() + dispatcherContext.getSourceName(); + source.append("import ").append(sourceClassFqn).append(";\n"); + + // Import all generated metrics classes + for (CodeGenModel metric : dispatcherContext.getMetrics()) { + String metricsClassFqn = oalDefine.getDynamicMetricsClassPackage() + metric.getMetricsName() + "Metrics"; + source.append("import ").append(metricsClassFqn).append(";\n"); + } + source.append("\n"); + + // Class declaration with generic type + source.append("public class ").append(className) + .append(" implements SourceDispatcher<").append(dispatcherContext.getSourceName()).append("> {\n\n"); + + // doMetrics methods for each metric + for (CodeGenModel metric : dispatcherContext.getMetrics()) { + StringWriter methodEntity = new StringWriter(); + try { + configuration.getTemplate("dispatcher/doMetrics.ftl").process(metric, methodEntity); + source.append(" ").append(methodEntity.toString().trim()).append("\n\n"); + } catch (Exception e) { + throw new OALCompileException( + "Failed to generate doMetrics for " + metric.getMetricsName() + ": " + e.getMessage(), e); + } + } + + // dispatch method + StringWriter dispatchMethod = new StringWriter(); + try { + configuration.getTemplate("dispatcher/dispatch.ftl").process(dispatcherContext, dispatchMethod); + source.append(" ").append(dispatchMethod.toString().trim()).append("\n\n"); + } catch (Exception e) { + throw new OALCompileException("Failed to generate dispatch method: " + e.getMessage(), e); + } + + source.append("}\n"); + return source.toString(); + } +} diff --git a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/OALSourceGenerationTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/OALSourceGenerationTest.java new file mode 100644 index 0000000000..12472bd779 --- /dev/null +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/OALSourceGenerationTest.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oal.v2.generator; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javassist.ClassPool; +import javassist.CtClass; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oal.v2.model.MetricDefinition; +import org.apache.skywalking.oal.v2.parser.OALScriptParserV2; +import org.apache.skywalking.oap.server.core.analysis.SourceDecoratorManager; +import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Generates Java source files to target/generated-test-sources/oal for all OAL scripts. + * + * <p>This test serves two purposes: + * <ol> + * <li>Generate readable Java source files for debugging and documentation</li> + * <li>Verify that generated sources are consistent with bytecode loaded into JVM</li> + * </ol> + * + * <p>The generated sources use the exact same FreeMarker templates as the bytecode generator, + * ensuring 100% consistency between source files and runtime-generated classes. + * + * <p>Output directory: target/generated-test-sources/oal/ + * <ul> + * <li>metrics/ - Generated metrics classes</li> + * <li>builders/ - Generated builder classes</li> + * <li>dispatchers/ - Generated dispatcher classes</li> + * </ul> + */ +@Slf4j +public class OALSourceGenerationTest { + + private static final String SOURCE_PACKAGE = "org.apache.skywalking.oap.server.core.source."; + private static final String METRICS_PACKAGE = "org.apache.skywalking.oap.server.core.source.oal.rt.metrics."; + + private static final String[] OAL_SCRIPTS = { + "oal/core.oal", + "oal/java-agent.oal", + "oal/browser.oal", + "oal/mesh.oal", + "oal/tcp.oal", + "oal/dotnet-agent.oal", + "oal/ebpf.oal", + "oal/cilium.oal" + }; + + private static final String OUTPUT_PATH = "target/generated-test-sources/oal"; + + @BeforeAll + public static void setup() throws IOException { + // Create output directory: target/generated-test-sources/oal + Files.createDirectories(getOutputDirectory().resolve("metrics")); + Files.createDirectories(getOutputDirectory().resolve("builders")); + Files.createDirectories(getOutputDirectory().resolve("dispatchers")); + + // Initialize scopes and decorators + initializeScopes(); + } + + private static Path getOutputDirectory() { + return Path.of(OUTPUT_PATH); + } + + private static void initializeScopes() { + DefaultScopeDefine.Listener listener = new DefaultScopeDefine.Listener(); + + // Core sources + notifyClass(listener, "Service"); + notifyClass(listener, "ServiceInstance"); + notifyClass(listener, "Endpoint"); + notifyClass(listener, "ServiceRelation"); + notifyClass(listener, "ServiceInstanceRelation"); + notifyClass(listener, "EndpointRelation"); + notifyClass(listener, "DatabaseAccess"); + notifyClass(listener, "CacheAccess"); + notifyClass(listener, "MQAccess"); + notifyClass(listener, "MQEndpointAccess"); + + // JVM sources + notifyClass(listener, "ServiceInstanceJVMCPU"); + notifyClass(listener, "ServiceInstanceJVMMemory"); + notifyClass(listener, "ServiceInstanceJVMMemoryPool"); + notifyClass(listener, "ServiceInstanceJVMGC"); + notifyClass(listener, "ServiceInstanceJVMThread"); + notifyClass(listener, "ServiceInstanceJVMClass"); + + // CLR sources + notifyClass(listener, "ServiceInstanceCLRCPU"); + notifyClass(listener, "ServiceInstanceCLRGC"); + notifyClass(listener, "ServiceInstanceCLRThread"); + + // Browser sources (different package) + String browserPackage = "org.apache.skywalking.oap.server.core.browser.source."; + notifyClassFromPackage(listener, browserPackage, "BrowserAppTraffic"); + notifyClassFromPackage(listener, browserPackage, "BrowserAppPageTraffic"); + notifyClassFromPackage(listener, browserPackage, "BrowserAppSingleVersionTraffic"); + notifyClassFromPackage(listener, browserPackage, "BrowserAppPerf"); + notifyClassFromPackage(listener, browserPackage, "BrowserAppPagePerf"); + notifyClassFromPackage(listener, browserPackage, "BrowserAppSingleVersionPerf"); + notifyClassFromPackage(listener, browserPackage, "BrowserAppResourcePerf"); + notifyClassFromPackage(listener, browserPackage, "BrowserAppWebInteractionPerf"); + notifyClassFromPackage(listener, browserPackage, "BrowserAppWebVitalsPerf"); + notifyClassFromPackage(listener, browserPackage, "BrowserErrorLog"); + + // Mesh sources + notifyClass(listener, "ServiceMesh"); + notifyClass(listener, "ServiceMeshService"); + notifyClass(listener, "ServiceMeshServiceInstance"); + notifyClass(listener, "ServiceMeshServiceRelation"); + notifyClass(listener, "ServiceMeshServiceInstanceRelation"); + + // TCP sources + notifyClass(listener, "TCPService"); + notifyClass(listener, "TCPServiceInstance"); + notifyClass(listener, "TCPServiceRelation"); + notifyClass(listener, "TCPServiceInstanceRelation"); + + // eBPF sources + notifyClass(listener, "EBPFProfilingSchedule"); + + // Cilium sources + notifyClass(listener, "CiliumService"); + notifyClass(listener, "CiliumServiceInstance"); + notifyClass(listener, "CiliumEndpoint"); + notifyClass(listener, "CiliumServiceRelation"); + notifyClass(listener, "CiliumServiceInstanceRelation"); + + // K8s sources + notifyClass(listener, "K8SService"); + notifyClass(listener, "K8SServiceInstance"); + notifyClass(listener, "K8SEndpoint"); + notifyClass(listener, "K8SServiceRelation"); + notifyClass(listener, "K8SServiceInstanceRelation"); + + // Process sources + notifyClass(listener, "Process"); + notifyClass(listener, "ProcessRelation"); + + // Register decorators + registerDecorator("ServiceDecorator"); + registerDecorator("EndpointDecorator"); + registerDecorator("K8SServiceDecorator"); + registerDecorator("K8SEndpointDecorator"); + } + + private static void registerDecorator(String decoratorName) { + try { + Class<?> clazz = Class.forName(SOURCE_PACKAGE + decoratorName); + SourceDecoratorManager manager = new SourceDecoratorManager(); + manager.addIfAsSourceDecorator(clazz); + } catch (Exception e) { + log.debug("Decorator {} registration: {}", decoratorName, e.getMessage()); + } + } + + private static void notifyClass(DefaultScopeDefine.Listener listener, String className) { + notifyClassFromPackage(listener, SOURCE_PACKAGE, className); + } + + private static void notifyClassFromPackage(DefaultScopeDefine.Listener listener, String packageName, String className) { + try { + Class<?> clazz = Class.forName(packageName + className); + listener.notify(clazz); + } catch (Exception e) { + log.debug("Scope {} registration: {}", className, e.getMessage()); + } + } + + /** + * Generate source files for all OAL scripts and verify consistency with bytecode. + */ + @Test + public void generateAllOALSources() throws Exception { + int totalMetrics = 0; + int totalFiles = 0; + List<String> errors = new ArrayList<>(); + + for (String scriptPath : OAL_SCRIPTS) { + String oalContent = loadOALScript(scriptPath); + if (oalContent == null) { + log.warn("Script not found: {}, skipping", scriptPath); + continue; + } + + try { + GenerationResult result = generateSourcesForScript(scriptPath, oalContent); + totalMetrics += result.metricsCount; + totalFiles += result.filesWritten; + + log.info("{}: {} metrics, {} files written", scriptPath, result.metricsCount, result.filesWritten); + } catch (Exception e) { + errors.add(scriptPath + ": " + e.getMessage()); + log.error("Failed to generate sources for {}: {}", scriptPath, e.getMessage()); + } + } + + log.info("=== Source Generation Summary ==="); + log.info("Total metrics processed: {}", totalMetrics); + log.info("Total files written: {}", totalFiles); + log.info("Output directory: {}", getOutputDirectory().toAbsolutePath()); + + if (!errors.isEmpty()) { + log.error("Errors encountered:"); + errors.forEach(e -> log.error(" - {}", e)); + } + + assertTrue(totalMetrics > 0, "Should generate sources for at least some metrics"); + assertTrue(totalFiles > 0, "Should write at least some files"); + assertTrue(errors.isEmpty(), "All scripts should generate successfully: " + errors); + } + + /** + * Test that generated sources produce identical bytecode as the bytecode generator. + * + * This verifies 100% consistency by comparing: + * 1. Method bodies generated from templates (same for both) + * 2. Field declarations and annotations + * 3. Class structure and inheritance + */ + @Test + public void verifySourceBytecodeConsistency() throws Exception { + // Use a simple OAL script for verification + String oal = "service_resp_time = from(Service.latency).longAvg();"; + + TestOALDefine define = new TestOALDefine(); + ClassPool classPool = new ClassPool(true); + + // Generate bytecode using V2ClassGenerator + V2ClassGenerator bytecodeGen = new V2ClassGenerator(define, classPool); + OALScriptParserV2 parser = OALScriptParserV2.parse(oal); + MetricDefinitionEnricher enricher = new MetricDefinitionEnricher(SOURCE_PACKAGE, METRICS_PACKAGE); + + MetricDefinition metric = parser.getMetrics().get(0); + CodeGenModel model = enricher.enrich(metric); + + CtClass bytecodeClass = bytecodeGen.generateMetricsCtClass(model); + byte[] bytecode = getBytecode(bytecodeClass); + + // Generate source using OALSourceGenerator + OALSourceGenerator sourceGen = new OALSourceGenerator(define); + sourceGen.setStorageBuilderFactory(new StorageBuilderFactory.Default()); + String source = sourceGen.generateMetricsSource(model); + + // Verify source contains all expected elements from FreeMarker templates + assertTrue(source.contains("public class ServiceRespTimeMetrics"), "Should have class declaration"); + assertTrue(source.contains("extends LongAvgMetrics"), "Should extend metrics function class"); + assertTrue(source.contains("implements WithMetadata"), "Should implement WithMetadata"); + assertTrue(source.contains("@Stream("), "Should have @Stream annotation"); + assertTrue(source.contains("@Column(name = \"entity_id\""), "Should have @Column annotation"); + assertTrue(source.contains("id0()"), "Should have id0() method from id.ftl"); + assertTrue(source.contains("public int hashCode()"), "Should have hashCode() method from hashCode.ftl"); + assertTrue(source.contains("serialize()"), "Should have serialize() method from serialize.ftl"); + + // Verify bytecode was generated + assertTrue(bytecode.length > 0, "Bytecode should be generated"); + + log.info("Source-bytecode consistency verified"); + log.info("Generated source size: {} chars", source.length()); + log.info("Generated bytecode size: {} bytes", bytecode.length); + + // Write the test source file for inspection + Path testOutput = getOutputDirectory().resolve("metrics/ServiceRespTimeMetrics.java"); + Files.writeString(testOutput, source); + log.info("Test source written to: {}", testOutput); + } + + private GenerationResult generateSourcesForScript(String scriptPath, String oalContent) throws Exception { + GenerationResult result = new GenerationResult(); + + TestOALDefine define = new TestOALDefine(); + OALSourceGenerator sourceGen = new OALSourceGenerator(define); + sourceGen.setStorageBuilderFactory(new StorageBuilderFactory.Default()); + MetricDefinitionEnricher enricher = new MetricDefinitionEnricher(SOURCE_PACKAGE, METRICS_PACKAGE); + + OALScriptParserV2 parser = OALScriptParserV2.parse(oalContent); + List<MetricDefinition> metrics = parser.getMetrics(); + result.metricsCount = metrics.size(); + + // Build dispatcher contexts (group by source) + Map<String, OALClassGeneratorV2.DispatcherContextV2> dispatcherContexts = new HashMap<>(); + List<CodeGenModel> models = new ArrayList<>(); + + for (MetricDefinition metric : metrics) { + CodeGenModel model = enricher.enrich(metric); + models.add(model); + + String sourceName = model.getSourceName(); + OALClassGeneratorV2.DispatcherContextV2 ctx = dispatcherContexts.computeIfAbsent(sourceName, name -> { + OALClassGeneratorV2.DispatcherContextV2 newCtx = new OALClassGeneratorV2.DispatcherContextV2(); + newCtx.setSourcePackage(SOURCE_PACKAGE); + newCtx.setSourceName(name); + newCtx.setPackageName(name.toLowerCase()); + newCtx.setSourceDecorator(model.getSourceDecorator()); + return newCtx; + }); + ctx.getMetrics().add(model); + } + + // Generate metrics sources + for (CodeGenModel model : models) { + String source = sourceGen.generateMetricsSource(model); + Path filePath = getOutputDirectory().resolve("metrics/" + model.getMetricsName() + "Metrics.java"); + Files.writeString(filePath, source, StandardCharsets.UTF_8); + result.filesWritten++; + + // Generate builder source + String builderSource = sourceGen.generateMetricsBuilderSource(model); + Path builderPath = getOutputDirectory().resolve("builders/" + model.getMetricsName() + "MetricsBuilder.java"); + Files.writeString(builderPath, builderSource, StandardCharsets.UTF_8); + result.filesWritten++; + } + + // Generate dispatcher sources + for (OALClassGeneratorV2.DispatcherContextV2 ctx : dispatcherContexts.values()) { + String source = sourceGen.generateDispatcherSource(ctx); + Path filePath = getOutputDirectory().resolve("dispatchers/" + ctx.getSourceName() + "Dispatcher.java"); + Files.writeString(filePath, source, StandardCharsets.UTF_8); + result.filesWritten++; + } + + return result; + } + + private String loadOALScript(String scriptPath) { + try (InputStream is = getClass().getClassLoader().getResourceAsStream(scriptPath)) { + if (is == null) { + return null; + } + return new String(is.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + return null; + } + } + + private byte[] getBytecode(CtClass ctClass) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ctClass.toBytecode(new DataOutputStream(baos)); + return baos.toByteArray(); + } + + private static class GenerationResult { + int metricsCount = 0; + int filesWritten = 0; + } + + private static class TestOALDefine extends OALDefine { + protected TestOALDefine() { + super("test.oal", SOURCE_PACKAGE); + } + } +} diff --git a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/SourceBytecodeConsistencyTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/SourceBytecodeConsistencyTest.java new file mode 100644 index 0000000000..78441c2178 --- /dev/null +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/SourceBytecodeConsistencyTest.java @@ -0,0 +1,387 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oal.v2.generator; + +import freemarker.template.Configuration; +import freemarker.template.Version; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtField; +import javassist.CtMethod; +import javassist.bytecode.AnnotationsAttribute; +import javassist.bytecode.annotation.Annotation; +import javassist.bytecode.annotation.IntegerMemberValue; +import javassist.bytecode.annotation.StringMemberValue; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oal.v2.model.MetricDefinition; +import org.apache.skywalking.oal.v2.parser.OALScriptParserV2; +import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.source.Service; +import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies that generated source files are 100% consistent with Javassist bytecode. + * + * <p>This test compares: + * <ul> + * <li>Method body strings from FreeMarker templates (used by both source and bytecode)</li> + * <li>Field declarations and annotations</li> + * <li>Class structure (inheritance, interfaces)</li> + * </ul> + * + * <p>The key insight: both OALSourceGenerator and OALClassGeneratorV2/V2ClassGenerator + * use the same FreeMarker templates. This test verifies template output is identical. + */ +@Slf4j +public class SourceBytecodeConsistencyTest { + + private static final String SOURCE_PACKAGE = "org.apache.skywalking.oap.server.core.source."; + private static final String METRICS_PACKAGE = "org.apache.skywalking.oap.server.core.source.oal.rt.metrics."; + private static final String[] TEMPLATE_METHODS = { + "id", "hashCode", "remoteHashCode", "equals", "serialize", "deserialize", "getMeta", "toHour", "toDay" + }; + + @BeforeAll + public static void setup() { + try { + DefaultScopeDefine.Listener listener = new DefaultScopeDefine.Listener(); + listener.notify(Service.class); + } catch (Exception e) { + // Already registered + } + } + + private static Configuration getTemplateConfiguration() { + Configuration config = new Configuration(new Version("2.3.28")); + config.setEncoding(Locale.ENGLISH, "UTF-8"); + config.setClassLoaderForTemplateLoading( + SourceBytecodeConsistencyTest.class.getClassLoader(), "/code-templates-v2"); + return config; + } + + /** + * Core consistency test: verify template outputs match between source and bytecode generators. + * + * Both generators use the same FreeMarker templates. This test: + * 1. Creates a CodeGenModel + * 2. Renders each template method + * 3. Verifies the output is used identically in both paths + */ + @Test + public void verifyTemplateOutputConsistency() throws Exception { + String oal = "service_resp_time = from(Service.latency).longAvg();"; + + TestOALDefine define = new TestOALDefine(); + MetricDefinitionEnricher enricher = new MetricDefinitionEnricher(SOURCE_PACKAGE, METRICS_PACKAGE); + + OALScriptParserV2 parser = OALScriptParserV2.parse(oal); + MetricDefinition metric = parser.getMetrics().get(0); + CodeGenModel model = enricher.enrich(metric); + + // Generate template output for each method + List<String> templateOutputs = new ArrayList<>(); + for (String method : TEMPLATE_METHODS) { + StringWriter writer = new StringWriter(); + getTemplateConfiguration().getTemplate("metrics/" + method + ".ftl").process(model, writer); + String output = writer.toString(); + templateOutputs.add(output); + + log.info("Template '{}' output length: {} chars", method, output.length()); + assertTrue(output.length() > 0, "Template " + method + " should produce output"); + } + + // Generate bytecode using V2ClassGenerator (same templates) + ClassPool classPool = new ClassPool(true); + V2ClassGenerator bytecodeGen = new V2ClassGenerator(define, classPool); + CtClass bytecodeClass = bytecodeGen.generateMetricsCtClass(model); + + // Verify bytecode class has all expected methods + for (String method : TEMPLATE_METHODS) { + String methodName = method.equals("id") ? "id0" : method; + CtMethod ctMethod = findMethod(bytecodeClass, methodName); + assertTrue(ctMethod != null, "Bytecode should have method: " + methodName); + } + + // Generate source using OALSourceGenerator (same templates) + OALSourceGenerator sourceGen = new OALSourceGenerator(define); + sourceGen.setStorageBuilderFactory(new StorageBuilderFactory.Default()); + String source = sourceGen.generateMetricsSource(model); + + // Verify source contains all template outputs + for (int i = 0; i < TEMPLATE_METHODS.length; i++) { + String templateOutput = templateOutputs.get(i).trim(); + // The source should contain the template output (method body) + assertTrue(source.contains(templateOutput), + "Source should contain template output for: " + TEMPLATE_METHODS[i]); + } + + log.info("Template consistency verified for {} methods", TEMPLATE_METHODS.length); + } + + /** + * Verify field consistency between source and bytecode. + */ + @Test + public void verifyFieldConsistency() throws Exception { + String oal = "service_resp_time = from(Service.latency).longAvg();"; + + TestOALDefine define = new TestOALDefine(); + MetricDefinitionEnricher enricher = new MetricDefinitionEnricher(SOURCE_PACKAGE, METRICS_PACKAGE); + ClassPool classPool = new ClassPool(true); + + OALScriptParserV2 parser = OALScriptParserV2.parse(oal); + MetricDefinition metric = parser.getMetrics().get(0); + CodeGenModel model = enricher.enrich(metric); + + // Generate bytecode + V2ClassGenerator bytecodeGen = new V2ClassGenerator(define, classPool); + CtClass bytecodeClass = bytecodeGen.generateMetricsCtClass(model); + + // Generate source + OALSourceGenerator sourceGen = new OALSourceGenerator(define); + sourceGen.setStorageBuilderFactory(new StorageBuilderFactory.Default()); + String source = sourceGen.generateMetricsSource(model); + + // Verify each field from model exists in both bytecode and source + for (CodeGenModel.SourceFieldV2 field : model.getFieldsFromSource()) { + // Check bytecode has field + CtField ctField = bytecodeClass.getDeclaredField(field.getFieldName()); + assertTrue(ctField != null, "Bytecode should have field: " + field.getFieldName()); + assertEquals(field.getType().getName(), ctField.getType().getName(), + "Field type should match for: " + field.getFieldName()); + + // Check source has field declaration + String fieldDecl = "private " + field.getType().getName() + " " + field.getFieldName(); + assertTrue(source.contains(fieldDecl), + "Source should have field declaration: " + fieldDecl); + + // Check source has @Column annotation + String columnAnnotation = "@Column(name = \"" + field.getColumnName() + "\""; + assertTrue(source.contains(columnAnnotation), + "Source should have @Column annotation for: " + field.getFieldName()); + } + + log.info("Field consistency verified for {} fields", model.getFieldsFromSource().size()); + } + + /** + * Verify class structure consistency. + */ + @Test + public void verifyClassStructureConsistency() throws Exception { + String oal = "service_resp_time = from(Service.latency).longAvg();"; + + TestOALDefine define = new TestOALDefine(); + MetricDefinitionEnricher enricher = new MetricDefinitionEnricher(SOURCE_PACKAGE, METRICS_PACKAGE); + ClassPool classPool = new ClassPool(true); + + OALScriptParserV2 parser = OALScriptParserV2.parse(oal); + MetricDefinition metric = parser.getMetrics().get(0); + CodeGenModel model = enricher.enrich(metric); + + // Generate bytecode + V2ClassGenerator bytecodeGen = new V2ClassGenerator(define, classPool); + CtClass bytecodeClass = bytecodeGen.generateMetricsCtClass(model); + + // Generate source + OALSourceGenerator sourceGen = new OALSourceGenerator(define); + sourceGen.setStorageBuilderFactory(new StorageBuilderFactory.Default()); + String source = sourceGen.generateMetricsSource(model); + + // Verify class name + String expectedClassName = model.getMetricsName() + "Metrics"; + assertEquals(expectedClassName, bytecodeClass.getSimpleName()); + assertTrue(source.contains("public class " + expectedClassName)); + + // Verify parent class + String expectedParent = model.getMetricsClassName(); + assertTrue(bytecodeClass.getSuperclass().getSimpleName().equals(expectedParent)); + assertTrue(source.contains("extends " + expectedParent)); + + // Verify interface + boolean hasWithMetadata = false; + for (CtClass iface : bytecodeClass.getInterfaces()) { + if (iface.getSimpleName().equals("WithMetadata")) { + hasWithMetadata = true; + break; + } + } + assertTrue(hasWithMetadata, "Bytecode should implement WithMetadata"); + assertTrue(source.contains("implements WithMetadata"), "Source should implement WithMetadata"); + + // Verify @Stream annotation content + assertTrue(source.contains("name = \"" + model.getTableName() + "\"")); + assertTrue(source.contains("scopeId = " + model.getSourceScopeId())); + + log.info("Class structure consistency verified"); + } + + /** + * Comprehensive annotation verification for class, fields, and methods. + * + * Verifies: + * - Class-level: @Stream (name, scopeId, builder, processor) + * - Field-level: @Column, @BanyanDB.SeriesID, @BanyanDB.ShardingKey, @ElasticSearch.EnableDocValues + * - Method-level: None expected (methods are generated without annotations) + * - Parameter-level: None expected + */ + @Test + public void verifyAllAnnotationsConsistency() throws Exception { + String oal = "service_resp_time = from(Service.latency).longAvg();"; + + TestOALDefine define = new TestOALDefine(); + MetricDefinitionEnricher enricher = new MetricDefinitionEnricher(SOURCE_PACKAGE, METRICS_PACKAGE); + ClassPool classPool = new ClassPool(true); + + OALScriptParserV2 parser = OALScriptParserV2.parse(oal); + MetricDefinition metric = parser.getMetrics().get(0); + CodeGenModel model = enricher.enrich(metric); + + // Generate bytecode + V2ClassGenerator bytecodeGen = new V2ClassGenerator(define, classPool); + CtClass bytecodeClass = bytecodeGen.generateMetricsCtClass(model); + + // Generate source + OALSourceGenerator sourceGen = new OALSourceGenerator(define); + sourceGen.setStorageBuilderFactory(new StorageBuilderFactory.Default()); + String source = sourceGen.generateMetricsSource(model); + + // ========== Class-level @Stream annotation ========== + AnnotationsAttribute classAnnotations = (AnnotationsAttribute) + bytecodeClass.getClassFile().getAttribute(AnnotationsAttribute.visibleTag); + Annotation streamAnnotation = classAnnotations.getAnnotation( + "org.apache.skywalking.oap.server.core.analysis.Stream"); + + // Verify @Stream in bytecode + assertTrue(streamAnnotation != null, "Bytecode should have @Stream annotation"); + String streamName = ((StringMemberValue) streamAnnotation.getMemberValue("name")).getValue(); + int streamScopeId = ((IntegerMemberValue) streamAnnotation.getMemberValue("scopeId")).getValue(); + assertEquals(model.getTableName(), streamName, "@Stream.name should match"); + assertEquals(model.getSourceScopeId(), streamScopeId, "@Stream.scopeId should match"); + + // Verify @Stream in source + assertTrue(source.contains("@Stream("), "Source should have @Stream annotation"); + assertTrue(source.contains("name = \"" + model.getTableName() + "\""), + "Source @Stream.name should match"); + assertTrue(source.contains("scopeId = " + model.getSourceScopeId()), + "Source @Stream.scopeId should match"); + assertTrue(source.contains("builder = " + model.getMetricsName() + "MetricsBuilder.class"), + "Source @Stream.builder should match"); + assertTrue(source.contains("processor = MetricsStreamProcessor.class"), + "Source @Stream.processor should match"); + + log.info("Class-level @Stream annotation verified"); + + // ========== Field-level annotations ========== + for (CodeGenModel.SourceFieldV2 field : model.getFieldsFromSource()) { + CtField ctField = bytecodeClass.getDeclaredField(field.getFieldName()); + AnnotationsAttribute fieldAnnotations = (AnnotationsAttribute) + ctField.getFieldInfo().getAttribute(AnnotationsAttribute.visibleTag); + + // @Column annotation + Annotation columnAnnotation = fieldAnnotations.getAnnotation( + "org.apache.skywalking.oap.server.core.storage.annotation.Column"); + assertTrue(columnAnnotation != null, + "Field " + field.getFieldName() + " should have @Column annotation in bytecode"); + String columnName = ((StringMemberValue) columnAnnotation.getMemberValue("name")).getValue(); + assertEquals(field.getColumnName(), columnName, + "@Column.name should match for field " + field.getFieldName()); + + // Verify @Column in source + assertTrue(source.contains("@Column(name = \"" + field.getColumnName() + "\""), + "Source should have @Column for field " + field.getFieldName()); + + // @BanyanDB.SeriesID for ID fields + if (field.isID()) { + Annotation seriesIdAnnotation = fieldAnnotations.getAnnotation( + "org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB$SeriesID"); + assertTrue(seriesIdAnnotation != null, + "ID field " + field.getFieldName() + " should have @BanyanDB.SeriesID in bytecode"); + + // Verify in source + assertTrue(source.contains("@BanyanDB.SeriesID(index = 0)"), + "Source should have @BanyanDB.SeriesID for ID field"); + + // @ElasticSearch.EnableDocValues for ID fields + Annotation docValuesAnnotation = fieldAnnotations.getAnnotation( + "org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch$EnableDocValues"); + assertTrue(docValuesAnnotation != null, + "ID field " + field.getFieldName() + " should have @ElasticSearch.EnableDocValues in bytecode"); + + assertTrue(source.contains("@ElasticSearch.EnableDocValues"), + "Source should have @ElasticSearch.EnableDocValues for ID field"); + } + + // @BanyanDB.ShardingKey for sharding key fields + if (field.isShardingKey()) { + Annotation shardingKeyAnnotation = fieldAnnotations.getAnnotation( + "org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB$ShardingKey"); + assertTrue(shardingKeyAnnotation != null, + "Sharding key field " + field.getFieldName() + " should have @BanyanDB.ShardingKey in bytecode"); + int shardingIdx = ((IntegerMemberValue) shardingKeyAnnotation.getMemberValue("index")).getValue(); + assertEquals(field.getShardingKeyIdx(), shardingIdx, + "@BanyanDB.ShardingKey.index should match for field " + field.getFieldName()); + + // Verify in source + assertTrue(source.contains("@BanyanDB.ShardingKey(index = " + field.getShardingKeyIdx() + ")"), + "Source should have @BanyanDB.ShardingKey for sharding key field"); + } + } + + log.info("Field-level annotations verified for {} fields", model.getFieldsFromSource().size()); + + // ========== Method-level annotations ========== + // Generated methods don't have annotations (template methods are plain Java) + for (CtMethod method : bytecodeClass.getDeclaredMethods()) { + AnnotationsAttribute methodAnnotations = (AnnotationsAttribute) + method.getMethodInfo().getAttribute(AnnotationsAttribute.visibleTag); + // No method annotations expected + if (methodAnnotations != null) { + log.debug("Method {} has annotations: {}", method.getName(), methodAnnotations.getAnnotations().length); + } + } + + log.info("All annotations consistency verified successfully"); + } + + private CtMethod findMethod(CtClass ctClass, String methodName) { + for (CtMethod method : ctClass.getDeclaredMethods()) { + if (method.getName().equals(methodName)) { + return method; + } + } + return null; + } + + private static class TestOALDefine extends OALDefine { + protected TestOALDefine() { + super("test.oal", SOURCE_PACKAGE); + } + } +}
