[ASTERIXDB-1976][FUN] Invoke code generation from asterix-runtime module - user model changes: no - storage format changes: no - interface changes: no
Details: - Invoke function code generation from asterix-runtime module instead of asterix-algebra module Change-Id: I2bb9a2cef1d346242c65d76378fbbf0e18a94bfe Reviewed-on: https://asterix-gerrit.ics.uci.edu/1870 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> BAD: Jenkins <[email protected]> Reviewed-by: Yingyi Bu <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/274f2967 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/274f2967 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/274f2967 Branch: refs/heads/master Commit: 274f2967185acc1dd047944213e1bf9c68d1c099 Parents: 5ae0c1f Author: Dmitry Lychagin <[email protected]> Authored: Wed Jul 5 20:31:12 2017 -0700 Committer: Dmitry Lychagin <[email protected]> Committed: Thu Jul 6 17:39:19 2017 -0700 ---------------------------------------------------------------------- asterixdb/asterix-algebra/pom.xml | 39 -- .../translator/util/FunctionCollection.java | 6 +- .../asterix/common/utils/CodeGenHelper.java | 93 +++++ .../pom.xml | 33 +- .../plugin/EvaluatorGeneratorMojo.java | 40 +- .../evaluators/staticcodegen/CodeGenUtil.java | 340 ++++++++++++++++ .../EvaluatorMissingCheckVisitor.java | 228 +++++++++++ .../EvaluatorNullCheckVisitor.java | 105 +++++ .../GatherEvaluatorCreationVisitor.java | 67 +++ .../GatherEvaluatorFactoryCreationVisitor.java | 66 +++ .../staticcodegen/GatherInnerClassVisitor.java | 56 +++ .../staticcodegen/MethodIdentifier.java | 57 +++ .../staticcodegen/RenameClassVisitor.java | 114 ++++++ asterixdb/asterix-runtime/pom.xml | 54 ++- .../evaluators/staticcodegen/CodeGenUtil.java | 405 ------------------- .../EvaluatorMissingCheckVisitor.java | 228 ----------- .../EvaluatorNullCheckVisitor.java | 105 ----- .../GatherEvaluatorCreationVisitor.java | 67 --- .../GatherEvaluatorFactoryCreationVisitor.java | 66 --- .../staticcodegen/GatherInnerClassVisitor.java | 56 --- .../staticcodegen/MethodIdentifier.java | 57 --- .../staticcodegen/RenameClassVisitor.java | 114 ------ 22 files changed, 1239 insertions(+), 1157 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-algebra/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/pom.xml b/asterixdb/asterix-algebra/pom.xml index 04f8d63..2cc9075 100644 --- a/asterixdb/asterix-algebra/pom.xml +++ b/asterixdb/asterix-algebra/pom.xml @@ -97,23 +97,6 @@ </execution> </executions> </plugin> - <plugin> - <groupId>org.apache.asterix</groupId> - <artifactId>asterix-evaluator-generator-maven-plugin</artifactId> - <version>${project.version}</version> - <configuration> - <evaluatorPackagePrefix>org.apache.asterix.runtime.evaluators</evaluatorPackagePrefix> - </configuration> - <executions> - <execution> - <id>generate-evaluator</id> - <phase>compile</phase> - <goals> - <goal>generate-evaluator</goal> - </goals> - </execution> - </executions> - </plugin> </plugins> <pluginManagement> <plugins> @@ -137,19 +120,6 @@ <ignore /> </action> </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.apache.asterix</groupId> - <artifactId>asterix-evaluator-generator-maven-plugin</artifactId> - <versionRange>(0.8.8-incubating,)</versionRange> - <goals> - <goal>javacc</goal> - </goals> - </pluginExecutionFilter> - <action> - <ignore /> - </action> - </pluginExecution> </pluginExecutions> </lifecycleMappingMetadata> </configuration> @@ -220,10 +190,6 @@ </dependency> <dependency> <groupId>org.apache.hyracks</groupId> - <artifactId>hyracks-util</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-dataflow-common</artifactId> </dependency> <dependency> @@ -262,11 +228,6 @@ <artifactId>hyracks-dataflow-std</artifactId> </dependency> <dependency> - <groupId>org.apache.asterix</groupId> - <artifactId>asterix-fuzzyjoin</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>org.apache.hyracks</groupId> <artifactId>algebricks-runtime</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/FunctionCollection.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/FunctionCollection.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/FunctionCollection.java index 92ca977..fa67e25 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/FunctionCollection.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/FunctionCollection.java @@ -23,6 +23,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; +import org.apache.asterix.common.utils.CodeGenHelper; import org.apache.asterix.om.functions.IFunctionDescriptorFactory; import org.apache.asterix.runtime.aggregates.collections.FirstElementAggregateDescriptor; import org.apache.asterix.runtime.aggregates.collections.ListifyAggregateDescriptor; @@ -324,7 +325,6 @@ import org.apache.asterix.runtime.evaluators.functions.temporal.UnixTimeFromDate import org.apache.asterix.runtime.evaluators.functions.temporal.UnixTimeFromTimeInMsDescriptor; import org.apache.asterix.runtime.evaluators.functions.temporal.YearMonthDurationGreaterThanComparatorDescriptor; import org.apache.asterix.runtime.evaluators.functions.temporal.YearMonthDurationLessThanComparatorDescriptor; -import org.apache.asterix.runtime.evaluators.staticcodegen.CodeGenUtil; import org.apache.asterix.runtime.runningaggregates.std.TidRunningAggregateDescriptor; import org.apache.asterix.runtime.unnestingfunctions.std.RangeDescriptor; import org.apache.asterix.runtime.unnestingfunctions.std.ScanCollectionDescriptor; @@ -731,8 +731,8 @@ public class FunctionCollection { */ private static IFunctionDescriptorFactory getGeneratedFunctionDescriptorFactory(Class<?> cl) { try { - String className = CodeGenUtil.getGeneratedFunctionDescriptorClassName(cl.getName(), - CodeGenUtil.DEFAULT_SUFFIX_FOR_GENERATED_CLASS); + String className = CodeGenHelper.getGeneratedClassName(cl.getName(), + CodeGenHelper.DEFAULT_SUFFIX_FOR_GENERATED_CLASS); Class<?> generatedCl = cl.getClassLoader().loadClass(className); Field factory = generatedCl.getDeclaredField(FACTORY); return (IFunctionDescriptorFactory) factory.get(null); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/CodeGenHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/CodeGenHelper.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/CodeGenHelper.java new file mode 100644 index 0000000..b8397f2 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/CodeGenHelper.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.common.utils; + +public final class CodeGenHelper { + + public final static String DEFAULT_SUFFIX_FOR_GENERATED_CLASS = "Gen"; + + private final static String DOLLAR = "$"; + + private final static String NESTED_CLASSNAME_PREFIX = "_"; + + public static String getGeneratedClassName(String originalClassName, String suffixForGeneratedClass) { + return toJdkStandardName(getGeneratedInternalClassName(originalClassName, suffixForGeneratedClass)); + } + + public static String getGeneratedInternalClassName(String originalClassName, String suffixForGeneratedClass) { + String originalFuncDescriptorClassInternalName = toInternalClassName(originalClassName); + return generateClassName(originalFuncDescriptorClassInternalName, suffixForGeneratedClass, 0); + } + + /** + * Gets the name of a generated class. + * + * @param originalClassName, + * the original class, i.e., the source of the generated class. + * @param suffix, + * the suffix for the generated class. + * @param counter, + * a counter that appearing at the end of the name of the generated class. + * @return the name of the generated class. + */ + public static String generateClassName(String originalClassName, String suffix, int counter) { + StringBuilder sb = new StringBuilder(); + int end = originalClassName.indexOf(DOLLAR); + if (end < 0) { + end = originalClassName.length(); + } + + String name = originalClassName.substring(0, end); + sb.append(name); + sb.append(DOLLAR); + sb.append(NESTED_CLASSNAME_PREFIX); + sb.append(suffix); + + if (counter > 0) { + sb.append(counter); + } + return sb.toString(); + } + + /** + * Converts an ASM class name to the JDK class naming format. + * + * @param name, + * a class name following the ASM convention. + * @return a "."-separated class name for JDK. + */ + public static String toJdkStandardName(String name) { + return name.replace("/", "."); + } + + /** + * Converts a JDK class name to the class naming format of ASM. + * + * @param name, + * a class name following the JDK convention. + * @return a "/"-separated class name assumed by ASM. + */ + public static String toInternalClassName(String name) { + return name.replace(".", "/"); + } + + private CodeGenHelper() { + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/pom.xml b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/pom.xml index a8ada2c..77ceb83 100644 --- a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/pom.xml +++ b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/pom.xml @@ -32,13 +32,30 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.10</version> + <configuration> + <ignoredUnusedDeclaredDependencies combine.children="append"> + <ignoredUnusedDeclaredDependency>org.apache.asterix:asterix-om:*</ignoredUnusedDeclaredDependency> + </ignoredUnusedDeclaredDependencies> + </configuration> + </plugin> + </plugins> + </build> <dependencies> <dependency> <groupId>org.apache.asterix</groupId> - <artifactId>asterix-runtime</artifactId> - <type>jar</type> + <artifactId>asterix-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-om</artifactId> <version>${project.version}</version> - <scope>compile</scope> </dependency> <dependency> <groupId>org.apache.maven</groupId> @@ -59,5 +76,15 @@ <artifactId>maven-model</artifactId> <version>3.3.9</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <version>3.5</version> + </dependency> + <dependency> + <groupId>org.ow2.asm</groupId> + <artifactId>asm-all</artifactId> + <version>5.1</version> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/plugin/EvaluatorGeneratorMojo.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/plugin/EvaluatorGeneratorMojo.java b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/plugin/EvaluatorGeneratorMojo.java index 888df55..74084af 100644 --- a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/plugin/EvaluatorGeneratorMojo.java +++ b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/plugin/EvaluatorGeneratorMojo.java @@ -22,21 +22,25 @@ package org.apache.asterix.runtime.evaluators.plugin; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; import java.util.Set; -import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor; +import org.apache.asterix.common.utils.CodeGenHelper; import org.apache.asterix.runtime.evaluators.staticcodegen.CodeGenUtil; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; import org.apache.maven.project.MavenProject; import org.reflections.Reflections; +import org.reflections.util.ConfigurationBuilder; /** * Statically generates null-handling byte code for scalar functions. * * @goal generate-evaluator - * @phase compile + * @phase process-classes */ public class EvaluatorGeneratorMojo extends AbstractMojo { @@ -62,23 +66,41 @@ public class EvaluatorGeneratorMojo extends AbstractMojo { @Override public void execute() throws MojoExecutionException, MojoFailureException { baseDir = project.getBuild().getDirectory() + File.separator + "classes"; + + URLClassLoader classLoader = null; try { - // Finds all sub-classes of AbstractScalarFunctionDynamicDescriptor with in the package - // org.apache.asterix.runtime.evaluators. - Reflections reflections = new Reflections(evaluatorPackagePrefix); - Set<Class<? extends AbstractScalarFunctionDynamicDescriptor>> allClasses = reflections - .getSubTypesOf(AbstractScalarFunctionDynamicDescriptor.class); + URI baseURI = new File(baseDir).toURI(); + classLoader = new URLClassLoader(new URL[] { baseURI.toURL() }, getClass().getClassLoader()); + + String superClassName = CodeGenHelper.toJdkStandardName(CodeGenUtil.DESCRIPTOR_SUPER_CLASS_NAME); + Class superClass = Class.forName(superClassName, false, classLoader); + + // Finds all sub-classes of the given root class within the specified package + ConfigurationBuilder config = ConfigurationBuilder.build(classLoader, evaluatorPackagePrefix); + String genSuffix = CodeGenHelper.DEFAULT_SUFFIX_FOR_GENERATED_CLASS + ".class"; + config.setInputsFilter(path -> path != null && !path.endsWith(genSuffix)); - // Generates byte code for all sub-classes of AbstractScalarFunctionDynamicDescriptor. + Reflections reflections = new Reflections(config); + Set<Class<?>> allClasses = reflections.getSubTypesOf(superClass); + + // Generates byte code for all sub-classes for (Class<?> cl : allClasses) { getLog().info("Generating byte code for " + cl.getName()); CodeGenUtil.generateScalarFunctionDescriptorBinary(evaluatorPackagePrefix, cl.getName(), - CodeGenUtil.DEFAULT_SUFFIX_FOR_GENERATED_CLASS, reflections.getClass().getClassLoader(), + CodeGenHelper.DEFAULT_SUFFIX_FOR_GENERATED_CLASS, classLoader, (name, bytes) -> writeFile(name, bytes)); } } catch (Exception e) { getLog().error(e); throw new MojoFailureException(e.toString()); + } finally { + if (classLoader != null) { + try { + classLoader.close(); + } catch (IOException e) { + getLog().error(e); + } + } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/CodeGenUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/CodeGenUtil.java b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/CodeGenUtil.java new file mode 100644 index 0000000..6d3feb7 --- /dev/null +++ b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/CodeGenUtil.java @@ -0,0 +1,340 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.evaluators.staticcodegen; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.asterix.common.utils.CodeGenHelper; +import org.apache.commons.lang3.tuple.Pair; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassWriter; + +/** + * A utility class that generates byte code for scalar function descriptors. + */ +public class CodeGenUtil { + + private final static String OBJECT_CLASS_NAME = "java/lang/Object"; + public final static String DESCRIPTOR_SUPER_CLASS_NAME = + "org/apache/asterix/runtime/evaluators/base/AbstractScalarFunctionDynamicDescriptor"; + private final static String EVALUATOR_FACTORY = "EvaluatorFactory"; + private final static String EVALUATOR = "Evaluator"; + private final static String INNER = "Inner"; + + /** + * The callback interface for a caller to determine what it needs to do for + * the generated class bytes. + */ + public static interface ClassByteCodeAction { + + /** + * Run a user-defined action for the generated class definition bytes. + * + * @param targetClassName, + * the name for the generated class. + * @param classDefinitionBytes, + * the class definition bytes. + * @throws IOException + */ + public void runAction(String targetClassName, byte[] classDefinitionBytes) throws IOException; + }; + + /** + * Generates the byte code for a scalar function descriptor. + * + * @param packagePrefix, + * the prefix of evaluators for code generation. + * @param originalFuncDescriptorClassName, + * the original class name of the function descriptor. + * @param suffixForGeneratedClass, + * the suffix for the generated class. + * @param action, + * the customized action for the generated class definition bytes. + * @throws IOException + * @throws ClassNotFoundException + */ + public static List<Pair<String, String>> generateScalarFunctionDescriptorBinary(String packagePrefix, + String originalFuncDescriptorClassName, String suffixForGeneratedClass, ClassLoader classLoader, + ClassByteCodeAction action) throws IOException, ClassNotFoundException { + String internalFuncDescriptorClassName = CodeGenHelper.toInternalClassName(originalFuncDescriptorClassName); + if (internalFuncDescriptorClassName.equals(DESCRIPTOR_SUPER_CLASS_NAME)) { + return Collections.emptyList(); + } + + String targetFuncDescriptorClassName = + CodeGenHelper.getGeneratedInternalClassName(internalFuncDescriptorClassName, suffixForGeneratedClass); + + // Adds the mapping of the old/new names of the function descriptor. + List<Pair<String, String>> nameMappings = new ArrayList<>(); + + // Generates code for super classes except java.lang.Object. + Class<?> evaluatorClass = + classLoader.loadClass(CodeGenHelper.toJdkStandardName(internalFuncDescriptorClassName)); + nameMappings.addAll(generateScalarFunctionDescriptorBinary(packagePrefix, + evaluatorClass.getSuperclass().getName(), suffixForGeneratedClass, classLoader, action)); + + nameMappings.add(Pair.of(internalFuncDescriptorClassName, targetFuncDescriptorClassName)); + nameMappings.add(Pair.of(CodeGenHelper.toJdkStandardName(internalFuncDescriptorClassName), + CodeGenHelper.toJdkStandardName(targetFuncDescriptorClassName))); + + // Gathers evaluator factory classes that are created in the function descriptor. + ClassReader reader = new ClassReader(getResourceStream(internalFuncDescriptorClassName, classLoader)); + GatherEvaluatorFactoryCreationVisitor evalFactoryCreationVisitor = + new GatherEvaluatorFactoryCreationVisitor(CodeGenHelper.toInternalClassName(packagePrefix)); + reader.accept(evalFactoryCreationVisitor, 0); + Set<String> evaluatorFactoryClassNames = evalFactoryCreationVisitor.getCreatedEvaluatorFactoryClassNames(); + + // Generates inner classes other than evaluator factories. + generateNonEvalInnerClasses(reader, evaluatorFactoryClassNames, nameMappings, suffixForGeneratedClass, + classLoader, action); + + // Generates evaluator factories that are created in the function descriptor. + int evalFactoryCounter = 0; + for (String evaluateFactoryClassName : evaluatorFactoryClassNames) { + generateEvaluatorFactoryClassBinary(packagePrefix, evaluateFactoryClassName, suffixForGeneratedClass, + evalFactoryCounter++, nameMappings, classLoader, action); + } + + // Transforms the function descriptor class and outputs the generated class binary. + ClassWriter writer = new ClassWriter(reader, 0); + RenameClassVisitor renamingVisitor = new RenameClassVisitor(writer, nameMappings); + reader.accept(renamingVisitor, 0); + action.runAction(targetFuncDescriptorClassName, writer.toByteArray()); + return nameMappings; + } + + /** + * Apply mappings for a class name. + * + * @param nameMappings, + * the mappings from existing class names to that of their generated counterparts. + * @param inputStr, + * the name of a class. + * @return the name of the generated counterpart class. + */ + static String applyMapping(List<Pair<String, String>> nameMappings, String inputStr) { + if (inputStr == null) { + return null; + } + String result = inputStr; + + // Applies name mappings in the reverse order, i.e., + // mapping recent added old/new name pairs first. + int index = nameMappings.size() - 1; + for (; index >= 0; --index) { + Pair<String, String> entry = nameMappings.get(index); + if (result.contains(entry.getLeft())) { + return result.replace(entry.getLeft(), entry.getRight()); + } + } + return result; + } + + /** + * Generates the byte code for an evaluator factory class. + * + * @param packagePrefix, + * the prefix of evaluators for code generation. + * @param originalEvaluatorFactoryClassName, + * the original evaluator factory class name. + * @param suffixForGeneratedClass, + * the suffix for the generated class. + * @param factoryCounter, + * the counter for the generated class. + * @param nameMappings, + * class names that needs to be rewritten in the generated byte code. + * @param classLoader, + * a class loader that has the original evaluator factory class in its resource paths. + * @param action, + * a user definition action for the generated byte code. + * @throws IOException + * @throws ClassNotFoundException + */ + private static void generateEvaluatorFactoryClassBinary(String packagePrefix, + String originalEvaluatorFactoryClassName, String suffixForGeneratedClass, int factoryCounter, + List<Pair<String, String>> nameMappings, ClassLoader classLoader, ClassByteCodeAction action) + throws IOException, ClassNotFoundException { + String internalEvaluatorFactoryClassName = CodeGenHelper.toInternalClassName(originalEvaluatorFactoryClassName); + String targetEvaluatorFactoryClassName = CodeGenHelper.generateClassName(internalEvaluatorFactoryClassName, + EVALUATOR_FACTORY + suffixForGeneratedClass, factoryCounter); + + // Adds the old/new names of the evaluator factory into the mapping. + nameMappings.add(Pair.of(internalEvaluatorFactoryClassName, targetEvaluatorFactoryClassName)); + nameMappings.add(Pair.of(CodeGenHelper.toJdkStandardName(internalEvaluatorFactoryClassName), + CodeGenHelper.toJdkStandardName(targetEvaluatorFactoryClassName))); + + // Gathers the class names of the evaluators that are created in the evaluator factory. + ClassReader reader = new ClassReader(getResourceStream(internalEvaluatorFactoryClassName, classLoader)); + GatherEvaluatorCreationVisitor evalCreationVisitor = + new GatherEvaluatorCreationVisitor(CodeGenHelper.toInternalClassName(packagePrefix)); + reader.accept(evalCreationVisitor, 0); + Set<String> evaluatorClassNames = evalCreationVisitor.getCreatedEvaluatorClassNames(); + + // Generates inner classes other than evaluators. + generateNonEvalInnerClasses(reader, evaluatorClassNames, nameMappings, suffixForGeneratedClass, classLoader, + action); + + // Generates code for all evaluators. + int evalCounter = 0; + for (String evaluateClassName : evaluatorClassNames) { + generateEvaluatorClassBinary(evaluateClassName, suffixForGeneratedClass, evalCounter++, nameMappings, + classLoader, action); + } + + // Transforms the evaluator factory class and outputs the generated class binary. + ClassWriter writer = new ClassWriter(reader, 0); + RenameClassVisitor renamingVisitor = new RenameClassVisitor(writer, nameMappings); + reader.accept(renamingVisitor, 0); + action.runAction(targetEvaluatorFactoryClassName, writer.toByteArray()); + } + + /** + * Generates the byte code for an evaluator class. + * + * @param originalEvaluatorClassName, + * the name of the original evaluator class. + * @param suffixForGeneratedClass, + * the suffix for the generated class. + * @param evalCounter, + * the counter for the generated class. + * @param nameMappings, + * class names that needs to be rewritten in the generated byte code. + * @param classLoader, + * a class loader that has the original evaluator factory class in its resource paths. + * @param action, + * a user definition action for the generated byte code. + * @throws IOException + * @throws ClassNotFoundException + */ + private static void generateEvaluatorClassBinary(String originalEvaluatorClassName, String suffixForGeneratedClass, + int evalCounter, List<Pair<String, String>> nameMappings, ClassLoader classLoader, + ClassByteCodeAction action) throws IOException, ClassNotFoundException { + // Convert class names. + String internalEvaluatorClassName = CodeGenHelper.toInternalClassName(originalEvaluatorClassName); + if (internalEvaluatorClassName.equals(OBJECT_CLASS_NAME)) { + return; + } + String targetEvaluatorClassName = CodeGenHelper.generateClassName(internalEvaluatorClassName, + EVALUATOR + suffixForGeneratedClass, evalCounter); + + // Generates code for super classes except java.lang.Object. + Class<?> evaluatorClass = classLoader.loadClass(CodeGenHelper.toJdkStandardName(internalEvaluatorClassName)); + generateEvaluatorClassBinary(evaluatorClass.getSuperclass().getName(), suffixForGeneratedClass, evalCounter, + nameMappings, classLoader, action); + + // Adds name mapping. + nameMappings.add(Pair.of(internalEvaluatorClassName, targetEvaluatorClassName)); + nameMappings.add(Pair.of(CodeGenHelper.toJdkStandardName(internalEvaluatorClassName), + CodeGenHelper.toJdkStandardName(targetEvaluatorClassName))); + + ClassReader firstPassReader = new ClassReader(getResourceStream(internalEvaluatorClassName, classLoader)); + // Generates inner classes other than the evaluator. + Set<String> excludedNames = new HashSet<>(); + for (Pair<String, String> entry : nameMappings) { + excludedNames.add(entry.getKey()); + } + generateNonEvalInnerClasses(firstPassReader, excludedNames, nameMappings, suffixForGeneratedClass, classLoader, + action); + + // Injects missing-handling byte code. + ClassWriter firstPassWriter = new ClassWriter(firstPassReader, 0); + EvaluatorMissingCheckVisitor missingHandlingVisitor = new EvaluatorMissingCheckVisitor(firstPassWriter); + firstPassReader.accept(missingHandlingVisitor, 0); + + ClassReader secondPassReader = new ClassReader(firstPassWriter.toByteArray()); + // Injects null-handling byte code and output the class binary. + // Since we're going to add jump instructions, we have to let the ClassWriter to + // automatically generate frames for JVM to verify the class. + ClassWriter secondPassWriter = + new ClassWriter(secondPassReader, ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); + RenameClassVisitor renamingVisitor = new RenameClassVisitor(secondPassWriter, nameMappings); + EvaluatorNullCheckVisitor nullHandlingVisitor = + new EvaluatorNullCheckVisitor(renamingVisitor, missingHandlingVisitor.getLastAddedLabel()); + secondPassReader.accept(nullHandlingVisitor, 0); + action.runAction(targetEvaluatorClassName, secondPassWriter.toByteArray()); + } + + /** + * Generates non-evaluator(-factory) inner classes defined in either a function descriptor + * or an evaluator factory. + * + * @param reader, + * the reader of the outer class. + * @param evalClassNames, + * the names of evaluator/evaluator-factory classes that shouldn't be generated in this + * method. + * @param nameMappings, + * class names that needs to be rewritten in the generated byte code. + * @param classLoader, + * a class loader that has the original evaluator factory class in its resource paths. + * @param action, + * a user definition action for the generated byte code. + * @throws IOException + */ + private static void generateNonEvalInnerClasses(ClassReader reader, Set<String> evalClassNames, + List<Pair<String, String>> nameMappings, String suffixForGeneratedClass, ClassLoader classLoader, + ClassByteCodeAction action) throws IOException { + // Gathers inner classes of the function descriptor. + GatherInnerClassVisitor innerClassVisitor = new GatherInnerClassVisitor(); + reader.accept(innerClassVisitor, 0); + Set<String> innerClassNames = innerClassVisitor.getInnerClassNames(); + innerClassNames.removeAll(evalClassNames); + + // Rewrites inner classes. + int counter = 0; + String suffix = INNER + suffixForGeneratedClass; + for (String innerClassName : innerClassNames) { + // adds name mapping. + String targetInnerClassName = CodeGenHelper.generateClassName(innerClassName, suffix, counter++); + nameMappings.add(Pair.of(innerClassName, targetInnerClassName)); + nameMappings.add(Pair.of(CodeGenHelper.toJdkStandardName(innerClassName), + CodeGenHelper.toJdkStandardName(targetInnerClassName))); + + // Renaming appearances of original class names. + ClassReader innerClassReader = new ClassReader(getResourceStream(innerClassName, classLoader)); + ClassWriter writer = new ClassWriter(innerClassReader, 0); + RenameClassVisitor renamingVisitor = new RenameClassVisitor(writer, nameMappings); + innerClassReader.accept(renamingVisitor, 0); + action.runAction(targetInnerClassName, writer.toByteArray()); + } + } + + /** + * Gets the input stream from a class file. + * + * @param className, + * the name of a class. + * @param classLoader, + * the corresponding class loader. + * @return the input stream. + */ + private static InputStream getResourceStream(String className, ClassLoader classLoader) { + return classLoader.getResourceAsStream(className.replace('.', '/') + ".class"); + } + + private CodeGenUtil() { + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/EvaluatorMissingCheckVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/EvaluatorMissingCheckVisitor.java b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/EvaluatorMissingCheckVisitor.java new file mode 100644 index 0000000..adedee5 --- /dev/null +++ b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/EvaluatorMissingCheckVisitor.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.runtime.evaluators.staticcodegen; + +import java.util.ArrayList; +import java.util.List; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.tree.AbstractInsnNode; +import org.objectweb.asm.tree.FieldInsnNode; +import org.objectweb.asm.tree.IincInsnNode; +import org.objectweb.asm.tree.InsnNode; +import org.objectweb.asm.tree.IntInsnNode; +import org.objectweb.asm.tree.VarInsnNode; + +/** + * This visitor adds missing-handling byte code into an evaluator class. + */ +public class EvaluatorMissingCheckVisitor extends ClassVisitor { + private static final String EVALUATE_DESC = "(Lorg/apache/hyracks/dataflow/common/data/" + + "accessors/IFrameTupleReference;Lorg/apache/hyracks/data/std/api/IPointable;)V"; + private static final String EVALUATE = "evaluate"; + private static final MethodIdentifier METHOD_IDENTIFIER = new MethodIdentifier(EVALUATE, EVALUATE_DESC, null); + private static final String TYPE_CHECKER_CLASS = "org/apache/asterix/runtime/evaluators/staticcodegen/TypeChecker"; + private static final String TYPE_CHECKER_DESC = "L" + TYPE_CHECKER_CLASS + ";"; + private static final String TYPE_CHECKER_NAME = "typeChecker"; + private static final String IS_MISSING = "isMissing"; + private static final String TYPECHECK_METHOD_DESC = "(Lorg/apache/hyracks/data/std/api/IPointable;" + + "Lorg/apache/hyracks/data/std/api/IPointable;)Z"; + private static final String CONSTRUCTOR = "<init>"; + private String className = null; + private Label lastAddedLabel = null; + + public EvaluatorMissingCheckVisitor(ClassVisitor downStreamVisitor) { + super(Opcodes.ASM5, downStreamVisitor); + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + if (cv != null) { + cv.visit(version, access, name, signature, superName, interfaces); + } + this.className = name; + } + + @Override + public void visitEnd() { + if (cv != null) { + cv.visitField(Opcodes.ACC_PRIVATE | Opcodes.ACC_FINAL, TYPE_CHECKER_NAME, TYPE_CHECKER_DESC, null, null); + cv.visitEnd(); + } + } + + @Override + public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { + MethodVisitor mv = cv.visitMethod(access, name, desc, signature, exceptions); + if (!METHOD_IDENTIFIER.equals(new MethodIdentifier(name, desc, signature)) && !name.equals(CONSTRUCTOR)) { + return mv; + } + if (name.equals(CONSTRUCTOR) && mv != null) { + return new ConstructorVisitor(Opcodes.ASM5, mv); + } + if (mv != null) { + return new InjectMissingCheckVisitor(Opcodes.ASM5, mv); + } + return null; + } + + // Obtains the last added label. + Label getLastAddedLabel() { + return lastAddedLabel; + } + + class ConstructorVisitor extends MethodVisitor { + + public ConstructorVisitor(int api, MethodVisitor mv) { + super(api, mv); + } + + @Override + public void visitInsn(int opcode) { + if (opcode != Opcodes.RETURN) { + mv.visitInsn(opcode); + return; + } + // Loads "this". + mv.visitVarInsn(Opcodes.ALOAD, 0); + // New TypeChecker. + mv.visitTypeInsn(Opcodes.NEW, TYPE_CHECKER_CLASS); + // Duplicate the top operand. + mv.visitInsn(Opcodes.DUP); + // Invoke the constructor of TypeChecker. + mv.visitMethodInsn(Opcodes.INVOKESPECIAL, TYPE_CHECKER_CLASS, CONSTRUCTOR, "()V", true); + // Putfield for the field typeChecker. + mv.visitFieldInsn(Opcodes.PUTFIELD, className, TYPE_CHECKER_NAME, TYPE_CHECKER_DESC); + // RETURN. + mv.visitInsn(Opcodes.RETURN); + } + } + + class InjectMissingCheckVisitor extends MethodVisitor { + + private FieldInsnNode fieldAccessNode = null; + private List<AbstractInsnNode> instructionsAfterFieldAccess = new ArrayList<>(); + private boolean updateToNextLabel = false; + + public InjectMissingCheckVisitor(int opcode, MethodVisitor mv) { + super(opcode, mv); + } + + @Override + public void visitFieldInsn(int opcode, String owner, String name, String desc) { + mv.visitFieldInsn(opcode, owner, name, desc); + fieldAccessNode = new FieldInsnNode(opcode, owner, name, desc); + instructionsAfterFieldAccess.clear(); + } + + @Override + public void visitIincInsn(int var, int increment) { + if (fieldAccessNode != null) { + instructionsAfterFieldAccess.add(new IincInsnNode(var, increment)); + } + super.visitIincInsn(var, increment); + } + + @Override + public void visitInsn(int opcode) { + if (fieldAccessNode != null) { + instructionsAfterFieldAccess.add(new InsnNode(opcode)); + } + super.visitInsn(opcode); + } + + @Override + public void visitIntInsn(int opcode, int operand) { + if (fieldAccessNode != null) { + instructionsAfterFieldAccess.add(new IntInsnNode(opcode, operand)); + } + super.visitIntInsn(opcode, operand); + } + + @Override + public void visitVarInsn(int opcode, int operand) { + if (fieldAccessNode != null) { + instructionsAfterFieldAccess.add(new VarInsnNode(opcode, operand)); + } + super.visitVarInsn(opcode, operand); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { + mv.visitMethodInsn(opcode, owner, name, desc, itf); + if (fieldAccessNode == null || !METHOD_IDENTIFIER.equals(new MethodIdentifier(name, desc, null))) { + return; + } + + // Loads the callee. + mv.visitVarInsn(Opcodes.ALOAD, 0); + mv.visitFieldInsn(Opcodes.GETFIELD, className, TYPE_CHECKER_NAME, TYPE_CHECKER_DESC); + + // Loads "this". + mv.visitVarInsn(Opcodes.ALOAD, 0); + // Replays the field access instruction. + fieldAccessNode.accept(mv); + + // Replays other instruction between the field access and the evaluator call. + for (AbstractInsnNode instruction : instructionsAfterFieldAccess) { + instruction.accept(mv); + } + + // Loads the result IPointable. + mv.visitVarInsn(Opcodes.ALOAD, 2); + + // Invokes the missing check method. + mv.visitMethodInsn(Opcodes.INVOKEVIRTUAL, TYPE_CHECKER_CLASS, IS_MISSING, TYPECHECK_METHOD_DESC, false); + lastAddedLabel = new Label(); + // Adds the if branch. + mv.visitJumpInsn(Opcodes.IFEQ, lastAddedLabel); + mv.visitInsn(Opcodes.RETURN); + mv.visitLabel(lastAddedLabel); + } + + @Override + public void visitLabel(Label label) { + if (updateToNextLabel) { + lastAddedLabel = label; + updateToNextLabel = false; + } + super.visitLabel(label); + } + + @Override + public void visitJumpInsn(int opcode, Label label) { + super.visitJumpInsn(opcode, label); + if (lastAddedLabel == null) { + return; + } + try { + if (label.getOffset() < lastAddedLabel.getOffset()) { + // Backward jump, i.e., loop. + updateToNextLabel = true; + } + } catch (IllegalStateException e) { + // Forward jump, the offset is not available. + } + } + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/EvaluatorNullCheckVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/EvaluatorNullCheckVisitor.java b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/EvaluatorNullCheckVisitor.java new file mode 100644 index 0000000..30f810b --- /dev/null +++ b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/EvaluatorNullCheckVisitor.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.runtime.evaluators.staticcodegen; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +/** + * This visitor adds null-handling byte code into an evaluator class. + */ +public class EvaluatorNullCheckVisitor extends ClassVisitor { + private final static String EVALUATE_DESC = "(Lorg/apache/hyracks/dataflow/common/data/accessors/" + + "IFrameTupleReference;Lorg/apache/hyracks/data/std/api/IPointable;)V"; + private final static String EVALUATE = "evaluate"; + private final static MethodIdentifier METHOD_IDENTIFIER = new MethodIdentifier(EVALUATE, EVALUATE_DESC, null); + private final static String TYPE_CHECKER_CLASS = "org/apache/asterix/runtime/evaluators/staticcodegen/" + + "TypeChecker"; + private final static String TYPE_CHECKER_DESC = "L" + TYPE_CHECKER_CLASS + ";"; + private final static String TYPE_CHECKER_NAME = "typeChecker"; + private final static String IS_NULL = "isNull"; + private final static String TYPECHECK_METHOD_DESC = "(Lorg/apache/hyracks/data/std/api/IPointable;)Z"; + private String className = null; + private final Label lastAddedLabel; + + public EvaluatorNullCheckVisitor(ClassVisitor downStreamVisitor, Label lastAddedLabel) { + super(Opcodes.ASM5, downStreamVisitor); + this.lastAddedLabel = lastAddedLabel; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + if (cv != null) { + cv.visit(version, access, name, signature, superName, interfaces); + } + this.className = name; + } + + @Override + public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { + MethodVisitor mv = cv.visitMethod(access, name, desc, signature, exceptions); + if (!METHOD_IDENTIFIER.equals(new MethodIdentifier(name, desc, signature))) { + return mv; + } + if (mv != null) { + return new InjectNullCheckVisitor(Opcodes.ASM5, mv); + } + return null; + } + + // Obtains the last added label. + Label getLastAddedLabel() { + return lastAddedLabel; + } + + class InjectNullCheckVisitor extends MethodVisitor { + + public InjectNullCheckVisitor(int api, MethodVisitor mv) { + super(api, mv); + } + + @Override + public void visitLabel(Label label) { + // Emits the label. + mv.visitLabel(label); + + // Injects null-handling after the last missing-handling byte code. + if (lastAddedLabel == null || lastAddedLabel.getOffset() != label.getOffset()) { + return; + } + + // Loads the callee. + mv.visitVarInsn(Opcodes.ALOAD, 0); + mv.visitFieldInsn(Opcodes.GETFIELD, className, TYPE_CHECKER_NAME, TYPE_CHECKER_DESC); + + // Loads the result IPointable. + mv.visitVarInsn(Opcodes.ALOAD, 2); + + // Invokes the null check method. + mv.visitMethodInsn(Opcodes.INVOKEVIRTUAL, TYPE_CHECKER_CLASS, IS_NULL, TYPECHECK_METHOD_DESC, false); + Label notNull = new Label(); + // Adds the if branch. + mv.visitJumpInsn(Opcodes.IFEQ, notNull); + mv.visitInsn(Opcodes.RETURN); + mv.visitLabel(notNull); + } + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherEvaluatorCreationVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherEvaluatorCreationVisitor.java b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherEvaluatorCreationVisitor.java new file mode 100644 index 0000000..17f60ee --- /dev/null +++ b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherEvaluatorCreationVisitor.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.evaluators.staticcodegen; + +import java.util.HashSet; +import java.util.Set; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +/** + * This visitor gathers all created evaluators in an evaluator factory. + */ +public class GatherEvaluatorCreationVisitor extends ClassVisitor { + + private static final String METHOD_NAME = "createScalarEvaluator"; + private Set<String> createdEvaluatorClassNames = new HashSet<>(); + private String ownerPrefix; + + public GatherEvaluatorCreationVisitor(String ownerPrefix) { + super(Opcodes.ASM5); + this.ownerPrefix = ownerPrefix; + } + + @Override + public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { + if (!name.equals(METHOD_NAME)) { + return null; + } + return new MethodVisitor(Opcodes.ASM5, null) { + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { + if (opcode != Opcodes.INVOKESPECIAL) { + return; + } + if (owner.startsWith(ownerPrefix)) { + createdEvaluatorClassNames.add(owner); + } + } + }; + + } + + public Set<String> getCreatedEvaluatorClassNames() { + return createdEvaluatorClassNames; + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherEvaluatorFactoryCreationVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherEvaluatorFactoryCreationVisitor.java b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherEvaluatorFactoryCreationVisitor.java new file mode 100644 index 0000000..eeeb313 --- /dev/null +++ b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherEvaluatorFactoryCreationVisitor.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.evaluators.staticcodegen; + +import java.util.HashSet; +import java.util.Set; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +/** + * This visitor gathers all created evaluator factories in a scalar function descriptor. + */ +public class GatherEvaluatorFactoryCreationVisitor extends ClassVisitor { + + private static final String METHOD_NAME = "createEvaluatorFactory"; + private final Set<String> createdEvaluatorFactoryClassNames = new HashSet<>(); + private String ownerPrefix; + + public GatherEvaluatorFactoryCreationVisitor(String ownerPrefix) { + super(Opcodes.ASM5); + this.ownerPrefix = ownerPrefix; + } + + @Override + public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { + if (!name.equals(METHOD_NAME)) { + return null; + } + return new MethodVisitor(Opcodes.ASM5, null) { + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { + if (opcode != Opcodes.INVOKESPECIAL) { + return; + } + if (owner.startsWith(ownerPrefix)) { + createdEvaluatorFactoryClassNames.add(owner); + } + } + }; + } + + public Set<String> getCreatedEvaluatorFactoryClassNames() { + return createdEvaluatorFactoryClassNames; + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherInnerClassVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherInnerClassVisitor.java b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherInnerClassVisitor.java new file mode 100644 index 0000000..031707b --- /dev/null +++ b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/GatherInnerClassVisitor.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.evaluators.staticcodegen; + +import java.util.HashSet; +import java.util.Set; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Opcodes; + +/** + * This class gathers all inner classes defined in a class. + */ +public class GatherInnerClassVisitor extends ClassVisitor { + + private final Set<String> innerClassNames = new HashSet<>(); + private String className = null; + + public GatherInnerClassVisitor() { + super(Opcodes.ASM5); + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + className = name; + } + + @Override + public void visitInnerClass(String name, String outerName, String innerName, int access) { + if ((className == null || !name.equals(className)) + && ((access & Opcodes.ACC_PUBLIC) == 0 || (access & Opcodes.ACC_STATIC) == 0)) { + innerClassNames.add(name); + } + } + + public Set<String> getInnerClassNames() { + return innerClassNames; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/MethodIdentifier.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/MethodIdentifier.java b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/MethodIdentifier.java new file mode 100644 index 0000000..c37b93e --- /dev/null +++ b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/MethodIdentifier.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.evaluators.staticcodegen; + +/** + * The data structure that can uniquely identify a method. + */ +public class MethodIdentifier { + + private final String name; + private final String desc; + private final String signature; + + public MethodIdentifier(String name, String desc, String signature) { + this.name = name == null ? "" : name; + this.desc = desc == null ? "" : desc; + this.signature = signature == null ? "" : signature; + } + + @Override + public int hashCode() { + return name.hashCode() * desc.hashCode() * signature.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof MethodIdentifier)) { + return false; + } + MethodIdentifier methodIdentifier = (MethodIdentifier) o; + return name.equals(methodIdentifier.name) && desc.equals(methodIdentifier.desc) + && signature.equals(methodIdentifier.signature); + } + + @Override + public String toString() { + return name + ":" + desc + ":" + signature; + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/RenameClassVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/RenameClassVisitor.java b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/RenameClassVisitor.java new file mode 100644 index 0000000..4396fd6 --- /dev/null +++ b/asterixdb/asterix-maven-plugins/asterix-evaluator-generator-maven-plugin/src/main/java/org/apache/asterix/runtime/evaluators/staticcodegen/RenameClassVisitor.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.evaluators.staticcodegen; + +import java.util.List; + +import org.apache.commons.lang3.tuple.Pair; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.FieldVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +/** + * This visitor replaces all the appearances of original class names with + * new (generated) class names, according to an input name mapping. + */ +public class RenameClassVisitor extends ClassVisitor { + + private final List<Pair<String, String>> nameMapping; + + public RenameClassVisitor(ClassVisitor downStreamVisitor, List<Pair<String, String>> nameMapping) { + super(Opcodes.ASM5, downStreamVisitor); + this.nameMapping = nameMapping; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + super.visit(version, access, applyMapping(name), signature, applyMapping(superName), interfaces); + } + + @Override + public void visitOuterClass(String owner, String name, String desc) { + // Skips outer class descriptions. + } + + @Override + public void visitInnerClass(String name, String outerName, String innerName, int access) { + if ((access & Opcodes.ACC_PUBLIC) != 0 && (access & Opcodes.ACC_STATIC) != 0) { + super.visitInnerClass(name, outerName, innerName, access); + } + } + + @Override + public FieldVisitor visitField(int access, String name, String desc, String signature, Object value) { + return cv.visitField(access, name, applyMapping(desc), applyMapping(signature), value); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { + MethodVisitor mv = cv.visitMethod(access, name, applyMapping(desc), applyMapping(signature), exceptions); + if (mv != null) { + return new MethodVisitor(Opcodes.ASM5, mv) { + + @Override + public void visitFieldInsn(int opcode, String owner, String name, String desc) { + mv.visitFieldInsn(opcode, applyMapping(owner), applyMapping(name), applyMapping(desc)); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { + mv.visitMethodInsn(opcode, applyMapping(owner), name, applyMapping(desc), itf); + } + + @Override + public void visitLocalVariable(String name, String desc, String signature, Label start, Label end, + int index) { + mv.visitLocalVariable(name, applyMapping(desc), applyMapping(signature), start, end, index); + } + + @Override + public void visitFrame(int type, int nLocal, Object[] local, int nStack, Object[] stack) { + if (local != null) { + for (int index = 0; index < local.length; ++index) { + if (local[index] instanceof String) { + local[index] = applyMapping((String) local[index]); + } + } + } + mv.visitFrame(type, nLocal, local, nStack, stack); + } + + @Override + public void visitTypeInsn(int opcode, String type) { + mv.visitTypeInsn(opcode, applyMapping(type)); + } + + }; + } + return null; + } + + private String applyMapping(String inputStr) { + return CodeGenUtil.applyMapping(nameMapping, inputStr); + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/274f2967/asterixdb/asterix-runtime/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/pom.xml b/asterixdb/asterix-runtime/pom.xml index 7f1fabd..6092cc5 100644 --- a/asterixdb/asterix-runtime/pom.xml +++ b/asterixdb/asterix-runtime/pom.xml @@ -35,6 +35,55 @@ <comments>A business-friendly OSS license</comments> </license> </licenses> + <build> + <plugins> + <plugin> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-evaluator-generator-maven-plugin</artifactId> + <version>${project.version}</version> + <configuration> + <evaluatorPackagePrefix>org.apache.asterix.runtime.evaluators</evaluatorPackagePrefix> + </configuration> + <executions> + <execution> + <id>generate-evaluator</id> + <phase>process-classes</phase> + <goals> + <goal>generate-evaluator</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.eclipse.m2e</groupId> + <artifactId>lifecycle-mapping</artifactId> + <version>1.0.0</version> + <configuration> + <lifecycleMappingMetadata> + <pluginExecutions> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-evaluator-generator-maven-plugin</artifactId> + <versionRange>(0.8.8-incubating,)</versionRange> + <goals> + <goal>generate-evaluator</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore /> + </action> + </pluginExecution> + </pluginExecutions> + </lifecycleMappingMetadata> + </configuration> + </plugin> + </plugins> + </pluginManagement> + </build> <dependencies> <dependency> <groupId>org.apache.asterix</groupId> @@ -135,11 +184,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm-all</artifactId> - <version>5.1</version> - </dependency> - <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </dependency>
