This is an automated email from the ASF dual-hosted git repository. wusheng pushed a commit to branch rt-oal in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 594c4426e510ef2ac3cb3dce2b3839180b40e417 Author: Wu Sheng <[email protected]> AuthorDate: Tue Jul 16 17:58:57 2019 +0800 Part of metrics class generation --- oap-server/oal-rt/pom.xml | 11 +- .../org/apache/skywalking/oal/rt/OALRuntime.java | 194 ++++++++++++++++++++- .../apache/skywalking/oal/rt/meta/MetaReader.java | 8 + .../skywalking/oal/rt/output/FileGenerator.java | 2 +- .../code-templates/metrics/deserialize.ftl | 27 +++ .../resources/code-templates/metrics/equals.ftl | 25 +++ .../resources/code-templates/metrics/getMeta.ftl | 3 + .../resources/code-templates/metrics/hashCode.ftl | 14 ++ .../main/resources/code-templates/metrics/id.ftl | 13 ++ .../code-templates/metrics/remoteHashCode.ftl | 13 ++ .../resources/code-templates/metrics/serialize.ftl | 26 +++ oap-server/pom.xml | 6 + .../oap/server/core/CoreModuleProvider.java | 9 +- 13 files changed, 341 insertions(+), 10 deletions(-) diff --git a/oap-server/oal-rt/pom.xml b/oap-server/oal-rt/pom.xml index 409347e..2ce0efc 100644 --- a/oap-server/oal-rt/pom.xml +++ b/oap-server/oal-rt/pom.xml @@ -17,7 +17,8 @@ ~ --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <parent> <artifactId>oap-server</artifactId> <groupId>org.apache.skywalking</groupId> @@ -46,5 +47,13 @@ <groupId>org.freemarker</groupId> <artifactId>freemarker</artifactId> </dependency> + <dependency> + <groupId>org.javassist</groupId> + <artifactId>javassist</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> </dependencies> </project> \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java index 0511120..cacc049 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java @@ -18,24 +18,75 @@ package org.apache.skywalking.oal.rt; +import freemarker.template.Configuration; +import freemarker.template.Version; import java.io.FileNotFoundException; import java.io.IOException; import java.io.Reader; +import java.io.StringWriter; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Locale; +import javassist.CannotCompileException; +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtField; +import javassist.CtNewMethod; +import javassist.NotFoundException; +import javassist.bytecode.AnnotationsAttribute; +import javassist.bytecode.ClassFile; +import javassist.bytecode.ClassFilePrinter; +import javassist.bytecode.ConstPool; +import javassist.bytecode.annotation.Annotation; +import javassist.bytecode.annotation.ClassMemberValue; +import javassist.bytecode.annotation.IntegerMemberValue; +import javassist.bytecode.annotation.StringMemberValue; import org.apache.skywalking.oal.rt.meta.MetaReader; import org.apache.skywalking.oal.rt.meta.MetaSettings; +import org.apache.skywalking.oal.rt.output.AllDispatcherContext; +import org.apache.skywalking.oal.rt.output.DispatcherContext; +import org.apache.skywalking.oal.rt.output.FileGenerator; +import org.apache.skywalking.oal.rt.parser.AnalysisResult; +import org.apache.skywalking.oal.rt.parser.MetricsHolder; import org.apache.skywalking.oal.rt.parser.OALScripts; import org.apache.skywalking.oal.rt.parser.ScriptParser; +import org.apache.skywalking.oal.rt.parser.SourceColumn; import org.apache.skywalking.oal.rt.parser.SourceColumnsFactory; +import org.apache.skywalking.oap.server.core.analysis.Stream; import org.apache.skywalking.oap.server.core.oal.rt.OALEngine; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.apache.skywalking.oap.server.library.util.ResourceUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * OAL Runtime is the class generation engine, which load the generated classes from OAL scrip definitions. + * OAL Runtime is the class generation engine, which load the generated classes from OAL scrip definitions. This runtime + * is loaded dynamically. * * @author wusheng */ public class OALRuntime implements OALEngine { + private static final Logger logger = LoggerFactory.getLogger(OALRuntime.class); + + private static final Charset CLASS_FILE_CHARSET = Charset.forName("UTF-8"); + private static final String METRICS_FUNCTION_PACKAGE = "org.apache.skywalking.oap.server.core.analysis.metrics."; + private static final String DYNAMIC_METRICS_CLASS_PACKAGE = "org.apache.skywalking.oal.rt.metrics."; + private static final String WITH_METADATA_INTERFACE = "org.apache.skywalking.oap.server.core.analysis.metrics.WithMetadata"; + private static final String METRICS_STREAM_PROCESSOR = "org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor"; + private static final String[] METRICS_CLASS_METHODS = {"id", "hashCode", "remoteHashCode", "equals", "serialize", "deserialize", "getMeta"}; + private final ClassPool classPool; + private Configuration configuration; + private AllDispatcherContext allDispatcherContext; + + public OALRuntime() { + classPool = ClassPool.getDefault(); + configuration = new Configuration(new Version("2.3.28")); + configuration.setEncoding(Locale.ENGLISH, "UTF-8"); + configuration.setClassLoaderForTemplateLoading(FileGenerator.class.getClassLoader(), "/code-templates"); + allDispatcherContext = new AllDispatcherContext(); + } + @Override public void start(ClassLoader currentClassLoader) throws ModuleStartException { Reader read; try { @@ -49,18 +100,153 @@ public class OALRuntime implements OALEngine { SourceColumnsFactory.setSettings(metaSettings); try { + MetricsHolder.init(); + } catch (IOException e) { + throw new ModuleStartException("load metrics functions error.", e); + } + + try { read = ResourceUtils.read("official_analysis.oal"); } catch (FileNotFoundException e) { throw new ModuleStartException("Can't locate official_analysis.oal", e); } - ScriptParser scriptParser = null; + OALScripts oalScripts; try { - scriptParser = ScriptParser.createFromFile(read); - OALScripts oalScripts = scriptParser.parse(); + ScriptParser scriptParser = ScriptParser.createFromFile(read); + oalScripts = scriptParser.parse(); } catch (IOException e) { throw new ModuleStartException("OAL script parse analysis failure.", e); } + this.generateClassAtRuntime(oalScripts); + } + + private void generateClassAtRuntime(OALScripts oalScripts) { + List<AnalysisResult> metricsStmts = oalScripts.getMetricsStmts(); + metricsStmts.forEach(this::buildDispatcherContext); + + metricsStmts.forEach(this::generateMetricsClass); + + } + + private void generateMetricsClass(AnalysisResult metricsStmt) { + CtClass parentMetricsClass = null; + try { + parentMetricsClass = classPool.get(METRICS_FUNCTION_PACKAGE + metricsStmt.getMetricsClassName()); + } catch (NotFoundException e) { + logger.error("Can't find parent class for " + metricsStmt.getMetricsName() + ".", e); + return; + } + CtClass metricsClass = classPool.makeClass(metricsClassName(metricsStmt), parentMetricsClass); + try { + metricsClass.addInterface(classPool.get(WITH_METADATA_INTERFACE)); + } catch (NotFoundException e) { + logger.error("Can't find WithMetadata interface for " + metricsStmt.getMetricsName() + ".", e); + return; + } + + ClassFile metricsClassClassFile = metricsClass.getClassFile(); + ConstPool constPool = metricsClassClassFile.getConstPool(); + + /** + * Add fields with annotations. + * + * private ${sourceField.typeName} ${sourceField.fieldName}; + */ + for (SourceColumn field : metricsStmt.getFieldsFromSource()) { + try { + CtField newField = CtField.make("private " + field.getType().getName() + " " + field.getFieldName() + ";", metricsClass); + + metricsClass.addField(newField); + + metricsClass.addMethod(CtNewMethod.getter(field.getFieldGetter(), newField)); + metricsClass.addMethod(CtNewMethod.setter(field.getFieldSetter(), newField)); + + AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constPool, AnnotationsAttribute.visibleTag); + /** + * Add @Column(columnName = "${sourceField.columnName}") + */ + Annotation columnAnnotation = new Annotation(Column.class.getName(), constPool); + columnAnnotation.addMemberValue("columnName", new StringMemberValue(field.getColumnName(), constPool)); + annotationsAttribute.addAnnotation(columnAnnotation); + + if (field.isID()) { + /** + * Add @IDColumn + */ + Annotation idAnnotation = new Annotation(Column.class.getName(), constPool); + annotationsAttribute.addAnnotation(idAnnotation); + } + + newField.getFieldInfo().addAttribute(annotationsAttribute); + + } catch (CannotCompileException e) { + logger.error("Can't add field(including set/get) " + field.getFieldName() + " in " + metricsStmt.getMetricsName() + ".", e); + return; + } + } + + /** + * Generate methods + */ + for (String method : METRICS_CLASS_METHODS) { + StringWriter methodEntity = new StringWriter(); + try { + configuration.getTemplate("metrics/" + method + ".ftl").process(metricsStmt, methodEntity); + metricsClass.addMethod(CtNewMethod.make(methodEntity.toString(), metricsClass)); + } catch (Exception e) { + logger.error("Can't generate method " + method + " for " + metricsStmt.getMetricsName() + ".", e); + return; + } + } + + + /** + * Add following annotation to the metrics class + * + * at Stream(name = "${tableName}", scopeId = ${sourceScopeId}, builder = ${metricsName}Metrics.Builder.class, processor = MetricsStreamProcessor.class) + */ + AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constPool, AnnotationsAttribute.visibleTag); + Annotation streamAnnotation = new Annotation(Stream.class.getName(), constPool); + streamAnnotation.addMemberValue("name", new StringMemberValue(metricsStmt.getTableName(), constPool)); + streamAnnotation.addMemberValue("scopeId", new IntegerMemberValue(constPool, metricsStmt.getSourceScopeId())); + streamAnnotation.addMemberValue("builder", new ClassMemberValue(metricsBuilderClassName(metricsStmt), constPool)); + streamAnnotation.addMemberValue("processor", new ClassMemberValue(METRICS_STREAM_PROCESSOR, constPool)); + + annotationsAttribute.addAnnotation(streamAnnotation); + metricsClassClassFile.addAttribute(annotationsAttribute); + + try { + metricsClass.toClass(); + Class.forName(metricsClass.getName()); + } catch (CannotCompileException e) { + logger.error("Can't compile " + metricsStmt.getMetricsName() + ".", e); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + + ClassFilePrinter.print(metricsClassClassFile); + } + + private String metricsClassName(AnalysisResult metricsStmt) { + return DYNAMIC_METRICS_CLASS_PACKAGE + metricsStmt.getMetricsName() + "Metrics"; + } + + private String metricsBuilderClassName(AnalysisResult metricsStmt) { + return metricsStmt.getMetricsName() + "Metrics.Builder"; + } + + private void buildDispatcherContext(AnalysisResult metricsStmt) { + String sourceName = metricsStmt.getSourceName(); + + DispatcherContext context = allDispatcherContext.getAllContext().get(sourceName); + if (context == null) { + context = new DispatcherContext(); + context.setSource(sourceName); + context.setPackageName(sourceName.toLowerCase()); + allDispatcherContext.getAllContext().put(sourceName, context); + } + context.getMetrics().add(metricsStmt); } } diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/MetaReader.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/MetaReader.java index a8cb99c..9933922 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/MetaReader.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/MetaReader.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oal.rt.meta; +import java.io.InputStream; import java.io.Reader; import org.yaml.snakeyaml.Yaml; @@ -31,4 +32,11 @@ public class MetaReader { return settings; } + + public MetaSettings read(InputStream reader) { + Yaml yaml = new Yaml(); + MetaSettings settings = yaml.loadAs(reader, MetaSettings.class); + + return settings; + } } diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/output/FileGenerator.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/output/FileGenerator.java index cb228b1..671bc46 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/output/FileGenerator.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/output/FileGenerator.java @@ -80,7 +80,7 @@ public class FileGenerator { + result.getMetricsName() + suffix; } - void generateMetricsImplementor(AnalysisResult result, Writer output) throws IOException, TemplateException { + public void generateMetricsImplementor(AnalysisResult result, Writer output) throws IOException, TemplateException { configuration.getTemplate("MetricsImplementor.ftl").process(result, output); } diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/deserialize.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/deserialize.ftl new file mode 100644 index 0000000..20ee249 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/deserialize.ftl @@ -0,0 +1,27 @@ +public void deserialize(org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData remoteData) { + <#list serializeFields.stringFields as field> + ${field.setter}(remoteData.getDataStrings(${field?index})); + </#list> + + <#list serializeFields.longFields as field> + ${field.setter}(remoteData.getDataLongs(${field?index})); + </#list> + + <#list serializeFields.doubleFields as field> + ${field.setter}(remoteData.getDataDoubles(${field?index})); + </#list> + + <#list serializeFields.intFields as field> + ${field.setter}(remoteData.getDataIntegers(${field?index})); + </#list> + + <#list serializeFields.intLongValuePairListFields as field> + setDetailGroup(new org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValueArray(30)); + + Iterator<rg.apache.skywalking.oap.server.core.remote.grpc.proto.IntKeyLongValuePair> iterator = remoteData.getDataIntLongPairListList().iterator(); + while (iterator.hasNext()) { + IntKeyLongValuePair element = iterator.next(); + getDetailGroup().add(new IntKeyLongValue(element.getKey(), element.getValue())); + } + </#list> +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/equals.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/equals.ftl new file mode 100644 index 0000000..15f9e17 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/equals.ftl @@ -0,0 +1,25 @@ +public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + + org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics metrics = (org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics)obj; + <#list fieldsFromSource as sourceField> + <#if sourceField.isID()> + <#if sourceField.getTypeName() == "java.lang.String"> + if (!${sourceField.fieldName}.equals(metrics.${sourceField.fieldName})) + <#else> + if (${sourceField.fieldName} != metrics.${sourceField.fieldName}) + </#if> + return false; + </#if> + </#list> + + if (getTimeBucket() != metrics.getTimeBucket()) + return false; + + return true; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/getMeta.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/getMeta.ftl new file mode 100644 index 0000000..bc7f39e --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/getMeta.ftl @@ -0,0 +1,3 @@ +public org.apache.skywalking.oap.server.core.analysis.metrics.MetricsMetaInfo getMeta() { + return new org.apache.skywalking.oap.server.core.analysis.metrics.MetricsMetaInfo("${varName}", ${sourceScopeId}<#if (fieldsFromSource?size>0) ><#list fieldsFromSource as field><#if field.isID()>, ${field.fieldName}</#if></#list></#if>); +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/hashCode.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/hashCode.ftl new file mode 100644 index 0000000..ed901e7 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/hashCode.ftl @@ -0,0 +1,14 @@ +public int hashCode() { + int result = 17; + <#list fieldsFromSource as sourceField> + <#if sourceField.isID()> + <#if sourceField.getTypeName() == "java.lang.String"> + result = 31 * result + ${sourceField.fieldName}.hashCode(); + <#else> + result += Const.ID_SPLIT + ${sourceField.fieldName}; + </#if> + </#if> + </#list> + result = 31 * result + (int)getTimeBucket(); + return result; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/id.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/id.ftl new file mode 100644 index 0000000..00cd1c8 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/id.ftl @@ -0,0 +1,13 @@ +public String id() { + String splitJointId = String.valueOf(getTimeBucket()); + <#list fieldsFromSource as sourceField> + <#if sourceField.isID()> + <#if sourceField.getTypeName() == "java.lang.String"> + splitJointId += org.apache.skywalking.oap.server.core.Const.ID_SPLIT + ${sourceField.fieldName}; + <#else> + splitJointId += org.apache.skywalking.oap.server.core.Const.ID_SPLIT + String.valueOf(${sourceField.fieldName}); + </#if> + </#if> + </#list> + return splitJointId; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/remoteHashCode.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/remoteHashCode.ftl new file mode 100644 index 0000000..9d75ab3 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/remoteHashCode.ftl @@ -0,0 +1,13 @@ +public int remoteHashCode() { + int result = 17; + <#list fieldsFromSource as sourceField> + <#if sourceField.isID()> + <#if sourceField.getTypeName() == "java.lang.String"> + result = 31 * result + ${sourceField.fieldName}.hashCode(); + <#else> + result += org.apache.skywalking.oap.server.core.Const.ID_SPLIT + ${sourceField.fieldName}; + </#if> + </#if> + </#list> + return result; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/serialize.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/serialize.ftl new file mode 100644 index 0000000..c0c0ca1 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/serialize.ftl @@ -0,0 +1,26 @@ +public org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData.Builder serialize() { + org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData.Builder remoteBuilder = org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData.newBuilder(); + <#list serializeFields.stringFields as field> + remoteBuilder.addDataStrings(${field.getter}()); + </#list> + + <#list serializeFields.longFields as field> + remoteBuilder.addDataLongs(${field.getter}()); + </#list> + + <#list serializeFields.doubleFields as field> + remoteBuilder.addDataDoubles(${field.getter}()); + </#list> + + <#list serializeFields.intFields as field> + remoteBuilder.addDataIntegers(${field.getter}()); + </#list> + <#list serializeFields.intLongValuePairListFields as field> + Iterator<org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValue> iterator = getDetailGroup().iterator(); + while (iterator.hasNext()) { + remoteBuilder.addDataIntLongPairList(iterator.next().serialize()); + } + </#list> + + return remoteBuilder; +} \ No newline at end of file diff --git a/oap-server/pom.xml b/oap-server/pom.xml index 28803de..e8ec8fc 100644 --- a/oap-server/pom.xml +++ b/oap-server/pom.xml @@ -84,6 +84,7 @@ <etcd.version>v3.2.3</etcd.version> <antlr.version>4.7.1</antlr.version> <freemarker.version>2.3.28</freemarker.version> + <javaassist.version>3.25.0-GA</javaassist.version> </properties> <dependencies> @@ -419,6 +420,11 @@ <artifactId>freemarker</artifactId> <version>${freemarker.version}</version> </dependency> + <dependency> + <groupId>org.javassist</groupId> + <artifactId>javassist</artifactId> + <version>${javaassist.version}</version> + </dependency> </dependencies> </dependencyManagement> </project> diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index 4ed3648..adc3192 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -82,17 +82,18 @@ public class CoreModuleProvider extends ModuleProvider { @Override public void prepare() throws ServiceNotProvidedException, ModuleStartException { AnnotationScan scopeScan = new AnnotationScan(); scopeScan.registerListener(new DefaultScopeDefine.Listener()); - try { + scopeScan.scan(); OALEngineLoader.get().start(getClass().getClassLoader()); } catch (Exception e) { throw new ModuleStartException(e.getMessage(), e); } - scopeScan.registerListener(DisableRegister.INSTANCE); - scopeScan.registerListener(new DisableRegister.SingleDisableScanListener()); + AnnotationScan oalDisable = new AnnotationScan(); + oalDisable.registerListener(DisableRegister.INSTANCE); + oalDisable.registerListener(new DisableRegister.SingleDisableScanListener()); try { - scopeScan.scan(); + oalDisable.scan(); } catch (IOException e) { throw new ModuleStartException(e.getMessage(), e); }
