Repository: incubator-apex-core Updated Branches: refs/heads/devel-3 a198a622f -> aaa21a3be
Fixed remaining checkstyle issues. Set line length to 120. Set continuation indent for wrapping to 4. Added check for annotations being on separate lines. APEX-230, APEX-248, APEX-249, APEX-239 Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/d14ab1e5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/d14ab1e5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/d14ab1e5 Branch: refs/heads/devel-3 Commit: d14ab1e570cf76d6849109751354276176354c35 Parents: f35522b Author: Ilya Ganelin <[email protected]> Authored: Thu Nov 5 09:35:21 2015 -0500 Committer: Ilya Ganelin <[email protected]> Committed: Wed Nov 11 12:55:51 2015 -0800 ---------------------------------------------------------------------- .idea/codeStyleSettings.xml | 5 +- apex_checks.xml | 12 +- api/pom.xml | 2 +- .../java/com/datatorrent/api/AutoMetric.java | 2 +- .../main/java/com/datatorrent/api/Operator.java | 2 +- bufferserver/pom.xml | 2 +- checkstyle-suppressions.xml | 28 ++ common/pom.xml | 2 +- engine/pom.xml | 2 +- .../stram/engine/StreamingContainer.java | 2 +- .../stram/webapp/LogicalOperatorInfo.java | 1 - .../stram/webapp/OperatorDiscoverer.java | 1 - .../com/datatorrent/stram/webapp/TypeGraph.java | 460 ++++++++++--------- .../stram/webapp/TypeGraphFactory.java | 14 +- .../datatorrent/stram/webapp/asm/ASMUtil.java | 5 + .../stram/webapp/asm/ClassNodeType.java | 34 +- .../stram/webapp/asm/CompactUtil.java | 20 +- .../stram/webapp/asm/FastClassIndexReader.java | 340 ++++++++++++++ .../stram/webapp/OperatorDiscoveryTest.java | 1 + .../webapp/asm/FastClassIndexReaderTest.java | 109 +++++ pom.xml | 6 +- 21 files changed, 778 insertions(+), 272 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/.idea/codeStyleSettings.xml ---------------------------------------------------------------------- diff --git a/.idea/codeStyleSettings.xml b/.idea/codeStyleSettings.xml index 3143cfb..d436e44 100644 --- a/.idea/codeStyleSettings.xml +++ b/.idea/codeStyleSettings.xml @@ -72,13 +72,15 @@ <option name="ALIGN_MULTILINE_RESOURCES" value="false" /> <option name="ALIGN_MULTILINE_FOR" value="false" /> <option name="SPACE_AFTER_TYPE_CAST" value="false" /> + <option name="METHOD_CALL_CHAIN_WRAP" value="1" /> <option name="IF_BRACE_FORCE" value="3" /> <option name="DOWHILE_BRACE_FORCE" value="3" /> <option name="WHILE_BRACE_FORCE" value="3" /> <option name="FOR_BRACE_FORCE" value="3" /> + <option name="WRAP_LONG_LINES" value="true" /> <indentOptions> <option name="INDENT_SIZE" value="2" /> - <option name="CONTINUATION_INDENT_SIZE" value="2" /> + <option name="CONTINUATION_INDENT_SIZE" value="4" /> <option name="TAB_SIZE" value="2" /> <option name="LABEL_INDENT_SIZE" value="2" /> <option name="LABEL_INDENT_ABSOLUTE" value="true" /> @@ -103,4 +105,3 @@ <option name="USE_PER_PROJECT_SETTINGS" value="true" /> </component> </project> - http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/apex_checks.xml ---------------------------------------------------------------------- diff --git a/apex_checks.xml b/apex_checks.xml index 81fa09d..7701858 100644 --- a/apex_checks.xml +++ b/apex_checks.xml @@ -97,6 +97,10 @@ <message key="ws.notPreceded" value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/> </module> + <module name="LineLength"> + <property name="max" value="120"/> + </module> + <module name="OneStatementPerLine"/> <module name="MultipleVariableDeclarations"/> <module name="ArrayTypeStyle"/> @@ -140,10 +144,16 @@ <property name="caseIndent" value="2"/> <property name="throwsIndent" value="2"/> <property name="arrayInitIndent" value="2"/> - <property name="lineWrappingIndentation" value="2"/> + <property name="lineWrappingIndentation" value="4"/> <property name="forceStrictCondition" value="true"/> </module> + <module name="AnnotationLocation"> + <property name="allowSamelineMultipleAnnotations" value="false"/> + <property name="allowSamelineSingleParameterlessAnnotation" value="true"/> + <property name="allowSamelineParameterizedAnnotation" value="false"/> + </module> + <module name="ImportOrder"> <property name="groups" value="/^javax?\./,org,/org\.apache/,com,/com\.datatorrent/,*"/> <property name="ordered" value="true"/> http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/api/pom.xml ---------------------------------------------------------------------- diff --git a/api/pom.xml b/api/pom.xml index 2e357b2..d85a48c 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -89,7 +89,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> <configuration> - <maxAllowedViolations>49</maxAllowedViolations> + <maxAllowedViolations>103</maxAllowedViolations> </configuration> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/api/src/main/java/com/datatorrent/api/AutoMetric.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/com/datatorrent/api/AutoMetric.java b/api/src/main/java/com/datatorrent/api/AutoMetric.java index ddf99c6..1c1fb25 100644 --- a/api/src/main/java/com/datatorrent/api/AutoMetric.java +++ b/api/src/main/java/com/datatorrent/api/AutoMetric.java @@ -87,7 +87,7 @@ public @interface AutoMetric * <li>w - week</li> * <li>M - month</li> * <li>q - quarter</li> - * <li>y - year</li + * <li>y - year</li> * </ul> * * @return time buckets. http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/api/src/main/java/com/datatorrent/api/Operator.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/com/datatorrent/api/Operator.java b/api/src/main/java/com/datatorrent/api/Operator.java index c362159..eb69266 100644 --- a/api/src/main/java/com/datatorrent/api/Operator.java +++ b/api/src/main/java/com/datatorrent/api/Operator.java @@ -36,7 +36,7 @@ public interface Operator extends Component<OperatorContext> * processed data. This is the default mode. * <br /> * In AT_MOST_ONCE mode in case of failure, the operator will start with the tuples which are being sent at the time - * the failed operator is recovered. Unlike AT_LEAST_MOST once operator, it will not try to recover the tuples which + * the failed operator is recovered. Unlike AT_LEAST_ONCE operator, it will not try to recover the tuples which * may have arrived while operator was down. Typically you would want to mark operators AT_MOST_ONCE if it does not * materially impact your computation if a few tuples are omitted from the computation and the expected throughput is * most likely to consume all the resources available for the operator or the DAG. http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/bufferserver/pom.xml ---------------------------------------------------------------------- diff --git a/bufferserver/pom.xml b/bufferserver/pom.xml index bb631b5..b2c7f8d 100644 --- a/bufferserver/pom.xml +++ b/bufferserver/pom.xml @@ -51,7 +51,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> <configuration> - <maxAllowedViolations>83</maxAllowedViolations> + <maxAllowedViolations>165</maxAllowedViolations> </configuration> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/checkstyle-suppressions.xml ---------------------------------------------------------------------- diff --git a/checkstyle-suppressions.xml b/checkstyle-suppressions.xml new file mode 100644 index 0000000..7496011 --- /dev/null +++ b/checkstyle-suppressions.xml @@ -0,0 +1,28 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<!DOCTYPE suppressions PUBLIC + "-//Puppy Crawl//DTD Suppressions 1.0//EN" + "http://www.puppycrawl.com/dtds/suppressions_1_0.dtd";> + +<suppressions> + <suppress checks="RegexpMultiline" files="DTCli.java"/> +</suppressions> http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/common/pom.xml ---------------------------------------------------------------------- diff --git a/common/pom.xml b/common/pom.xml index 6d5c415..ed1a251 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -74,7 +74,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> <configuration> - <maxAllowedViolations>84</maxAllowedViolations> + <maxAllowedViolations>124</maxAllowedViolations> </configuration> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/pom.xml ---------------------------------------------------------------------- diff --git a/engine/pom.xml b/engine/pom.xml index 1513728..6c1d009 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -145,7 +145,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> <configuration> - <maxAllowedViolations>2072</maxAllowedViolations> + <maxAllowedViolations>2942</maxAllowedViolations> </configuration> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java b/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java index 1544c16..14e00a9 100644 --- a/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java +++ b/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java @@ -1445,7 +1445,7 @@ public class StreamingContainer extends YarnContainerMain try { signal.await(); } catch (InterruptedException ex) { - logger.debug("Activation of operators interruped.", ex); + logger.debug("Activation of operators interrupted.", ex); } for (ComponentContextPair<Stream, StreamContext> pair : newStreams.values()) { http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/main/java/com/datatorrent/stram/webapp/LogicalOperatorInfo.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/LogicalOperatorInfo.java b/engine/src/main/java/com/datatorrent/stram/webapp/LogicalOperatorInfo.java index e001f5a..9d8d648 100644 --- a/engine/src/main/java/com/datatorrent/stram/webapp/LogicalOperatorInfo.java +++ b/engine/src/main/java/com/datatorrent/stram/webapp/LogicalOperatorInfo.java @@ -18,7 +18,6 @@ */ package com.datatorrent.stram.webapp; -import com.datatorrent.stram.appdata.AppDataPushAgent; import java.util.*; import javax.xml.bind.annotation.*; import org.apache.commons.lang3.mutable.MutableInt; http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java b/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java index 55af0ae..7a24c51 100644 --- a/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java +++ b/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java @@ -375,7 +375,6 @@ public class OperatorDiscoverer typeGraph.trim(); - typeGraph.updatePortTypeInfoInTypeGraph(openJarFiles, openClassFiles); } finally { for (Entry<String, JarFile> entry : openJarFiles.entrySet()) { try { http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java index 2a55e8f..6cf36f3 100644 --- a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java +++ b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java @@ -22,27 +22,49 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; +import java.util.Queue; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.regex.Pattern; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.text.WordUtils; + import org.codehaus.jackson.map.deser.std.FromStringDeserializer; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; -import org.apache.xbean.asm5.ClassReader; -import org.apache.xbean.asm5.Opcodes; -import org.apache.xbean.asm5.tree.ClassNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.text.WordUtils; +import org.apache.xbean.asm5.ClassReader; +import org.apache.xbean.asm5.tree.ClassNode; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.google.common.collect.ImmutableSet; +import com.google.common.primitives.Primitives; + import com.datatorrent.api.Component; import com.datatorrent.api.InputOperator; import com.datatorrent.api.Operator; - import com.datatorrent.common.util.BaseOperator; import com.datatorrent.netlet.util.DTThrowable; import com.datatorrent.stram.webapp.asm.ClassNodeType; @@ -51,6 +73,7 @@ import com.datatorrent.stram.webapp.asm.CompactClassNode; import com.datatorrent.stram.webapp.asm.CompactFieldNode; import com.datatorrent.stram.webapp.asm.CompactMethodNode; import com.datatorrent.stram.webapp.asm.CompactUtil; +import com.datatorrent.stram.webapp.asm.FastClassIndexReader; import com.datatorrent.stram.webapp.asm.MethodSignatureVisitor; import com.datatorrent.stram.webapp.asm.Type; import com.datatorrent.stram.webapp.asm.Type.ArrayTypeNode; @@ -58,12 +81,6 @@ import com.datatorrent.stram.webapp.asm.Type.ParameterizedTypeNode; import com.datatorrent.stram.webapp.asm.Type.TypeNode; import com.datatorrent.stram.webapp.asm.Type.TypeVariableNode; import com.datatorrent.stram.webapp.asm.Type.WildcardTypeNode; -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.Serializer; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; -import com.google.common.collect.ImmutableSet; -import com.google.common.primitives.Primitives; /** * A graph data structure holds all type information and their relationship needed in app builder @@ -103,8 +120,9 @@ public class TypeGraph TypeGraphVertex parentVertex = typeGraph.get(parentClassName); TypeGraphVertex classVertex = typeGraph.get(subClassName); - if(parentVertex == null || classVertex == null) + if (parentVertex == null || classVertex == null) { return false; + } return TypeGraph.isAncestor(parentVertex, classVertex); } @@ -123,8 +141,9 @@ public class TypeGraph return false; } for (TypeGraphVertex vertex : tgv.ancestors) { - if (isAncestor(typeTgv, vertex)) + if (isAncestor(typeTgv, vertex)) { return true; + } } return false; } @@ -134,7 +153,8 @@ public class TypeGraph return typeGraph.get(typeName); } - enum UI_TYPE { + enum UI_TYPE + { LIST("List", Collection.class.getName()), @@ -204,8 +224,8 @@ public class TypeGraph { List<String> allTypes = TypeGraph.getAllAncestors(tgv, true); for (UI_TYPE type : UI_TYPE.values()) { - for (String assignable : type.allAssignableTypes){ - if(allTypes.contains(assignable)){ + for (String assignable : type.allAssignableTypes) { + if (allTypes.contains(assignable)) { return type; } } @@ -221,17 +241,19 @@ public class TypeGraph } } - public static List<String> getAllAncestors(TypeGraphVertex tgv, boolean include) { + public static List<String> getAllAncestors(TypeGraphVertex tgv, boolean include) + { List<String> result = new LinkedList<String>(); - if(include) { + if (include) { result.add(tgv.typeName); } getAllAncestors(tgv, result); return result; } - private static void getAllAncestors(TypeGraphVertex tgv, List<String> result) { - for(TypeGraphVertex an : tgv.ancestors){ + private static void getAllAncestors(TypeGraphVertex tgv, List<String> result) + { + for (TypeGraphVertex an : tgv.ancestors) { result.add(an.typeName); getAllAncestors(an, result); } @@ -245,39 +267,37 @@ public class TypeGraph private TypeGraphVertex addNode(InputStream input, String resName) throws IOException { try { - - ClassReader reader = new ClassReader(input); - ClassNode classN = new ClassNodeType(); - reader.accept(classN, ClassReader.SKIP_CODE); - CompactClassNode ccn = CompactUtil.compactClassNode(classN); - String typeName = classN.name.replace('/', '.'); - TypeGraphVertex tgv = null; - TypeGraphVertex ptgv = null; + + FastClassIndexReader fastClassIndexReader = new FastClassIndexReader(input); + String typeName = fastClassIndexReader.getName().replace('/', '.'); + TypeGraphVertex tgv; + TypeGraphVertex ptgv; if (typeGraph.containsKey(typeName)) { tgv = typeGraph.get(typeName); - tgv.setClassNode(ccn); - tgv.setJarName(resName); // If tgv was already populated for superclass/interface, jar name needs to be updated + tgv.setIsRealNode(true); + tgv.setJarName(resName); // If tgv was already populated for superclass/interface, jar name needs to be updated + tgv.setIsInstantiable(fastClassIndexReader.isInstantiable()); } else { - tgv = new TypeGraphVertex(typeName, resName, ccn); + tgv = new TypeGraphVertex(this, typeName, resName, true, fastClassIndexReader.isInstantiable()); typeGraph.put(typeName, tgv); } - String immediateP = reader.getSuperName(); + String immediateP = fastClassIndexReader.getSuperName(); if (immediateP != null) { immediateP = immediateP.replace('/', '.'); ptgv = typeGraph.get(immediateP); if (ptgv == null) { - ptgv = new TypeGraphVertex(immediateP, resName); + ptgv = new TypeGraphVertex(this, immediateP, resName); typeGraph.put(immediateP, ptgv); } tgv.ancestors.add(ptgv); ptgv.descendants.add(tgv); } - if (reader.getInterfaces() != null) { - for (String iface : reader.getInterfaces()) { + if (fastClassIndexReader.getInterfaces() != null) { + for (String iface : fastClassIndexReader.getInterfaces()) { iface = iface.replace('/', '.'); ptgv = typeGraph.get(iface); if (ptgv == null) { - ptgv = new TypeGraphVertex(iface, resName); + ptgv = new TypeGraphVertex(this, iface, resName); typeGraph.put(iface, ptgv); } tgv.ancestors.add(ptgv); @@ -304,75 +324,9 @@ public class TypeGraph return addNode(jar.getInputStream(jarEntry), jar.getName()); } - public void updatePortTypeInfoInTypeGraph(Map<String, JarFile> openJarFiles, - Map<String, File> openClassFiles) { - TypeGraphVertex tgv = typeGraph.get(Operator.class.getName()); - updatePortTypeInfoInTypeGraph(openJarFiles, openClassFiles, tgv); - } - - public void updatePortTypeInfoInTypeGraph(Map<String, JarFile> openJarFiles, - Map<String, File> openClassFiles, TypeGraphVertex tgv) { - if (tgv == null) - return; - - for (TypeGraphVertex operator : tgv.descendants) { - try { - String path = operator.getJarName(); - JarFile jar = openJarFiles.get(path); - if (jar != null) { - String jarEntryName = operator.getClassNode().getName() - .replace('.', '/') - + ".class"; - JarEntry jarEntry = jar.getJarEntry(jarEntryName); - if (jarEntry != null) { - updatePortInfo(operator, jar.getInputStream(jarEntry)); - } - } else { - File f = openClassFiles.get(path); - if (f != null && f.exists() && f.getName().endsWith("class")) { - updatePortInfo(operator, new FileInputStream(f)); - } - } - updatePortTypeInfoInTypeGraph(openJarFiles, openClassFiles, operator); - } catch (Exception e) { - DTThrowable.wrapIfChecked(e); - } - } - } - - private void updatePortInfo(TypeGraphVertex tgv, InputStream input) - throws IOException { - try { - ClassReader reader; - reader = new ClassReader(input); - ClassNodeType classN = new ClassNodeType(); - classN.setClassSignatureVisitor(tgv.getClassNode().getCsv()); - classN.setVisitFields(true); - reader.accept(classN, ClassReader.SKIP_CODE); - CompactClassNode ccn = tgv.getClassNode(); - CompactUtil.updateCompactClassPortInfo(classN, ccn); - List<CompactFieldNode> prunedFields = new LinkedList<CompactFieldNode>(); - TypeGraphVertex portVertex = typeGraph.get(Operator.Port.class.getName()); - for (CompactFieldNode field : ccn.getPorts()) { - TypeGraphVertex fieldVertex = typeGraph.get(field.getDescription()); - if(fieldVertex != null) { - if (isAncestor(portVertex, fieldVertex)) { - prunedFields.add(field); - } - } - } - ccn.setPorts(prunedFields); - - } finally { - if (input != null) { - input.close(); - } - } - } - private void updateInstantiableDescendants(TypeGraphVertex tgv) { - if(tgv.isInstantiable()){ + if (tgv.isInstantiable()) { tgv.allInstantiableDescendants.add(tgv); } for (TypeGraphVertex parent : tgv.ancestors) { @@ -468,13 +422,12 @@ public class TypeGraph { String n1 = o1.typeName; String n2 = o2.typeName; - if(n1.startsWith("java")){ + if (n1.startsWith("java")) { n1 = "0" + n1; } - if(n2.startsWith("java")){ + if (n2.startsWith("java")) { n2 = "0" + n2; } - return n1.compareTo(n2); } }); @@ -483,40 +436,59 @@ public class TypeGraph private final transient Set<TypeGraphVertex> descendants = new HashSet<TypeGraphVertex>(); + private transient TypeGraph owner; + // keep the jar file name for late fetching the detail information private String jarName; private boolean hasResource = false; + private boolean isRealNode = false; + + private boolean isInstantiable = false; + + @SuppressWarnings("unused") - private TypeGraphVertex(){ + private TypeGraphVertex() + { jarName = ""; typeName = ""; } - public TypeGraphVertex(String typeName, String jarName, CompactClassNode classNode) + public TypeGraphVertex(TypeGraph owner, String typeName, String jarName, boolean isRealNode, boolean isInstantiable) { - - this.jarName = jarName; this.typeName = typeName; - this.classNode = classNode; + this.jarName = jarName; + this.isRealNode = isRealNode; + this.isInstantiable = isInstantiable; + this.owner = owner; + } + + public TypeGraphVertex(TypeGraph owner, String typeName, String jarName) + { + this(owner, typeName, jarName, false, false); + } + + public void setOwner(TypeGraph owner) + { + this.owner = owner; + } + + public TypeGraph getOwner() + { + return owner; } public Set<TypeGraphVertex> getAncestors() { return ancestors; } + public int numberOfInstantiableDescendants() { return allInstantiableDescendants.size() + (isInstantiable() ? 1 : 0); } - public TypeGraphVertex(String typeName, String jarName) - { - this.typeName = typeName; - this.jarName = jarName; - } - public boolean hasResource() { return hasResource; @@ -529,33 +501,12 @@ public class TypeGraph public boolean isInstantiable() { - return JACKSON_INSTANTIABLE_CLASSES.contains(this.typeName) || (isPublicConcrete() && classNode.getDefaultConstructor() != null); + return isInstantiable || JACKSON_INSTANTIABLE_CLASSES.contains(this.typeName); } - private boolean isPublicConcrete() + public void setIsInstantiable(boolean isInstantiable) { - if (classNode == null) { - // If the class is not in the classpath - return false; - } - int opCode = getOpCode(); - - // if the class is neither abstract nor interface - // and the class is public - return ((opCode & (Opcodes.ACC_ABSTRACT | Opcodes.ACC_INTERFACE)) == 0) && ((opCode & Opcodes.ACC_PUBLIC) == Opcodes.ACC_PUBLIC); - } - - private int getOpCode() - { - List<CompactClassNode> icl = classNode.getInnerClasses(); - if (typeName.contains("$")) { - for (CompactClassNode innerClassNode : icl) { - if (innerClassNode.getName().replace('/', '.').equals(typeName)) { - return innerClassNode.getAccess(); - } - } - } - return classNode.getAccess(); + this.isInstantiable = isInstantiable; } /* @@ -575,18 +526,23 @@ public class TypeGraph @Override public boolean equals(Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null) + } + if (obj == null) { return false; - if (getClass() != obj.getClass()) + } + if (getClass() != obj.getClass()) { return false; - TypeGraphVertex other = (TypeGraphVertex) obj; + } + TypeGraphVertex other = (TypeGraphVertex)obj; if (typeName == null) { - if (other.typeName != null) + if (other.typeName != null) { return false; - } else if (!typeName.equals(other.typeName)) + } + } else if (!typeName.equals(other.typeName)) { return false; + } return true; } @@ -600,15 +556,80 @@ public class TypeGraph this.jarName = jarName; } - public CompactClassNode getClassNode() + /** + * The query on this vertex is possible to be called by multithread + * Thus make this method synchronized + * @return + */ + public synchronized CompactClassNode getOrLoadClassNode() { + if (classNode == null) { + //load the class first + try { + loadClass(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } return classNode; } - - public void setClassNode(CompactClassNode classNode) + + public void setIsRealNode(boolean isRealNode) { - this.classNode = classNode; + this.isRealNode = isRealNode; } + + public boolean isRealNode() + { + return isRealNode; + } + + private void loadClass() throws IOException + { + if (classNode != null) { + return; + } + for (TypeGraphVertex ancestor : getAncestors()) { + ancestor.loadClass(); + } + JarFile jarFile = null; + InputStream inputStream = null; + try { + if (jarName.endsWith(".jar")) { + JarFile jarfile = new JarFile(jarName); + inputStream = jarfile.getInputStream(jarfile.getEntry(typeName.replace('.', '/') + ".class")); + } else { + inputStream = new FileInputStream(jarName); + } + ClassReader reader = new ClassReader(inputStream); + ClassNode classN = new ClassNodeType(); + reader.accept(classN, ClassReader.SKIP_CODE); + CompactClassNode ccn = CompactUtil.compactClassNode(classN); + this.classNode = ccn; + + // update the port information if it is a Operator + if (owner.isAncestor(Operator.class.getName(), typeName)) { + // load ports if it is an Operator class + CompactUtil.updateCompactClassPortInfo(classN, ccn); + List<CompactFieldNode> prunedFields = new LinkedList<CompactFieldNode>(); + TypeGraphVertex portVertex = owner.getTypeGraphVertex(Operator.Port.class.getName()); + for (CompactFieldNode field : ccn.getPorts()) { + TypeGraphVertex fieldVertex = owner.getTypeGraphVertex(field.getDescription()); + if (fieldVertex != null) { + if (isAncestor(portVertex, fieldVertex)) { + prunedFields.add(field); + } + } + } + ccn.setPorts(prunedFields); + } + } finally { + IOUtils.closeQuietly(jarFile); + IOUtils.closeQuietly(inputStream); + } + + } + } /** @@ -621,7 +642,7 @@ public class TypeGraph public List<String> getInstantiableDescendants(String clazz, String filter, String packagePrefix, String startsWith) { TypeGraphVertex tgv = typeGraph.get(clazz); - if(tgv == null) { + if (tgv == null) { return null; } @@ -636,7 +657,8 @@ public class TypeGraph if (packagePrefix != null && !node.typeName.startsWith(packagePrefix)) { continue; } - if (startsWith != null && !typeName.substring(typeName.lastIndexOf('.') + 1).toLowerCase().startsWith(startsWith.toLowerCase())){ + if (startsWith != null && !typeName.substring(typeName.lastIndexOf('.') + 1).toLowerCase() + .startsWith(startsWith.toLowerCase())) { continue; } result.add(node.typeName); @@ -653,7 +675,7 @@ public class TypeGraph if (tgv == null) { return desc; } - CompactClassNode cn = tgv.classNode; + CompactClassNode cn = tgv.getOrLoadClassNode(); if (cn.isEnum()) { List<String> enumNames = cn.getEnumValues(); @@ -667,7 +689,7 @@ public class TypeGraph addClassPropertiesAndPorts(clazzName, desc); - if(tgv.hasResource()){ + if (tgv.hasResource()) { desc.put("hasResource", "true"); } else { desc.put("hasResource", "false"); @@ -677,7 +699,8 @@ public class TypeGraph } private Collection<JSONObject> getPortTypeInfo(String clazzName, - Map<Type, Type> typeReplacement, List<CompactFieldNode> ports) throws JSONException { + Map<Type, Type> typeReplacement, List<CompactFieldNode> ports) throws JSONException + { TypeGraphVertex tgv = typeGraph.get(clazzName); if (tgv == null) { return null; @@ -686,43 +709,45 @@ public class TypeGraph Collection<JSONObject> portInfo = new ArrayList<JSONObject>(); for (CompactFieldNode port : ports) { - Type fieldType = port.getFieldSignatureNode().getFieldType(); - Type t = fieldType; - if (fieldType instanceof ParameterizedTypeNode) { - // TODO: Right now getPortInfo assumes a single parameterized type - t = ((ParameterizedTypeNode) fieldType).getActualTypeArguments()[0]; - } else { - // TODO: Check behavior for Ports not using Default Input/output ports - TypeGraphVertex portVertex = typeGraph.get(port.getDescription()); - t = findTypeArgument(portVertex, typeReplacement); - LOG.debug("Field is of type {}", fieldType.getClass()); - } + Type fieldType = port.getFieldSignatureNode().getFieldType(); + Type t = fieldType; + if (fieldType instanceof ParameterizedTypeNode) { + // TODO: Right now getPortInfo assumes a single parameterized type + t = ((ParameterizedTypeNode)fieldType).getActualTypeArguments()[0]; + } else { + // TODO: Check behavior for Ports not using Default Input/output ports + TypeGraphVertex portVertex = typeGraph.get(port.getDescription()); + t = findTypeArgument(portVertex, typeReplacement); + LOG.debug("Field is of type {}", fieldType.getClass()); + } - JSONObject meta = new JSONObject(); - try { - meta.put("name", port.getName()); - setTypes(meta, t, typeReplacement); - portInfo.add(meta); - } catch (Exception e) { - DTThrowable.wrapIfChecked(e); - } + JSONObject meta = new JSONObject(); + try { + meta.put("name", port.getName()); + setTypes(meta, t, typeReplacement); + portInfo.add(meta); + } catch (Exception e) { + DTThrowable.wrapIfChecked(e); + } } return portInfo; } - public static Type getParameterizedTypeArgument(Type type) { + public static Type getParameterizedTypeArgument(Type type) + { if (type instanceof ParameterizedTypeNode) { - return ((ParameterizedTypeNode) type).getActualTypeArguments()[0]; + return ((ParameterizedTypeNode)type).getActualTypeArguments()[0]; } return null; } - private Type findTypeArgument(TypeGraphVertex tgv, - Map<Type, Type> typeReplacement) { - if (tgv == null) + private Type findTypeArgument(TypeGraphVertex tgv, Map<Type, Type> typeReplacement) + { + if (tgv == null) { return null; - ClassSignatureVisitor csv = tgv.getClassNode().getCsv(); + } + ClassSignatureVisitor csv = tgv.getOrLoadClassNode().getCsv(); Type superC = csv.getSuperClass(); addReplacement(superC, typeReplacement); @@ -757,14 +782,16 @@ public class TypeGraph public List<CompactFieldNode> getAllInputPorts(TypeGraphVertex tgv) { List<CompactFieldNode> ports = new ArrayList<CompactFieldNode>(); - if (tgv == null) + if (tgv == null) { return ports; - TypeGraphVertex portVertex = typeGraph.get(Operator.InputPort.class - .getName()); + } + TypeGraphVertex portVertex = typeGraph.get(Operator.InputPort.class.getName()); getAllPortsWithAncestor(portVertex, tgv, ports); - Collections.sort(ports, new Comparator<CompactFieldNode>() { + Collections.sort(ports, new Comparator<CompactFieldNode>() + { @Override - public int compare(CompactFieldNode a, CompactFieldNode b) { + public int compare(CompactFieldNode a, CompactFieldNode b) + { return a.getName().compareTo(b.getName()); } }); @@ -791,7 +818,7 @@ public class TypeGraph private void getAllPortsWithAncestor(TypeGraphVertex portVertex, TypeGraphVertex tgv, List<CompactFieldNode> ports) { - List<CompactFieldNode> fields = tgv.getClassNode().getPorts(); + List<CompactFieldNode> fields = tgv.getOrLoadClassNode().getPorts(); if (fields != null) { for (CompactFieldNode field : fields) { TypeGraphVertex fieldVertex = typeGraph.get(field.getDescription()); @@ -889,8 +916,8 @@ public class TypeGraph CompactClassNode exClass = null; // check if the class needs to be excluded for (String e : EXCLUDE_CLASSES) { - if(e.equals(tgv.getClassNode().getName())) { - exClass = tgv.getClassNode(); + if (e.equals(tgv.getOrLoadClassNode().getName())) { + exClass = tgv.getOrLoadClassNode(); break; } } @@ -914,17 +941,17 @@ public class TypeGraph } } } else { - if (tgv.getClassNode().getSetterMethods() != null) { - setters.addAll(tgv.getClassNode().getSetterMethods()); + if (tgv.getOrLoadClassNode().getSetterMethods() != null) { + setters.addAll(tgv.getOrLoadClassNode().getSetterMethods()); } - if (tgv.getClassNode().getGetterMethods() != null) { - getters.addAll(tgv.getClassNode().getGetterMethods()); + if (tgv.getOrLoadClassNode().getGetterMethods() != null) { + getters.addAll(tgv.getOrLoadClassNode().getGetterMethods()); } } TypeGraphVertex portVertex = typeGraph.get(Operator.Port.class.getName()); - List<CompactFieldNode> fields = tgv.getClassNode().getPorts(); - if(fields != null) { + List<CompactFieldNode> fields = tgv.getOrLoadClassNode().getPorts(); + if (fields != null) { for (CompactFieldNode field : fields) { TypeGraphVertex fieldVertex = typeGraph.get(field.getDescription()); if (isAncestor(portVertex, fieldVertex)) { @@ -933,15 +960,15 @@ public class TypeGraph } } - ClassSignatureVisitor csv = tgv.getClassNode().getCsv(); + ClassSignatureVisitor csv = tgv.getOrLoadClassNode().getCsv(); Type superC = csv.getSuperClass(); addReplacement(superC, typeReplacement); - if(csv.getInterfaces()!=null){ - for(Type it : csv.getInterfaces()){ + if (csv.getInterfaces() != null) { + for (Type it : csv.getInterfaces()) { addReplacement(it, typeReplacement); - }; + } } for (TypeGraphVertex ancestor : tgv.ancestors) { getPublicSetterGetterAndPorts(ancestor, setters, getters, typeReplacement, ports); @@ -950,9 +977,9 @@ public class TypeGraph private void addReplacement(Type superT, Map<Type, Type> typeReplacement) { - if(superT!=null && superT instanceof ParameterizedTypeNode){ + if (superT != null && superT instanceof ParameterizedTypeNode) { Type[] actualTypes = ((ParameterizedTypeNode)superT).getActualTypeArguments(); - List<TypeVariableNode> tvs = typeGraph.get(((ParameterizedTypeNode)superT).getTypeObj().getClassName()).getClassNode().getCsv().getTypeV(); + List<TypeVariableNode> tvs = typeGraph.get(((ParameterizedTypeNode)superT).getTypeObj().getClassName()).getOrLoadClassNode().getCsv().getTypeV(); int i = 0; for (TypeVariableNode typeVariableNode : tvs) { typeReplacement.put(typeVariableNode, actualTypes[i++]); @@ -978,7 +1005,7 @@ public class TypeGraph if (t instanceof WildcardTypeNode) { propJ.put("type", "?"); } else if (t instanceof TypeNode) { - TypeNode tn = (TypeNode) t; + TypeNode tn = (TypeNode)t; String typeS = tn.getTypeObj().getClassName(); propJ.put("type", typeS); UI_TYPE uiType = UI_TYPE.getEnumFor(typeS, typeGraph); @@ -1001,7 +1028,7 @@ public class TypeGraph } if (t instanceof ParameterizedTypeNode) { JSONArray jArray = new JSONArray(); - for (Type ttn : ((ParameterizedTypeNode) t).getActualTypeArguments()) { + for (Type ttn : ((ParameterizedTypeNode)t).getActualTypeArguments()) { JSONObject objJ = new JSONObject(); if (!stopRecursive) { setTypes(objJ, ttn, typeReplacement, visitedType); @@ -1016,18 +1043,18 @@ public class TypeGraph JSONArray jArray = new JSONArray(); - Type[] bounds = ((WildcardTypeNode) t).getUpperBounds(); - if(bounds!=null){ + Type[] bounds = ((WildcardTypeNode)t).getUpperBounds(); + if (bounds != null) { for (Type type : bounds) { jArray.put(type.toString()); } } typeBounds.put("upper", jArray); - bounds = ((WildcardTypeNode) t).getLowerBounds(); + bounds = ((WildcardTypeNode)t).getLowerBounds(); jArray = new JSONArray(); - if(bounds!=null){ + if (bounds != null) { for (Type type : bounds) { jArray.put(type.toString()); } @@ -1037,21 +1064,21 @@ public class TypeGraph propJ.put("typeBounds", typeBounds); } - if(t instanceof ArrayTypeNode){ + if (t instanceof ArrayTypeNode) { propJ.put("type", t.getByteString()); propJ.put("uiType", UI_TYPE.LIST.getName()); JSONObject jObj = new JSONObject(); if (!stopRecursive) { - setTypes(jObj, ((ArrayTypeNode) t).getActualArrayType(), typeReplacement, visitedType); + setTypes(jObj, ((ArrayTypeNode)t).getActualArrayType(), typeReplacement, visitedType); } propJ.put("itemType", jObj); } - if(t instanceof TypeVariableNode){ + if (t instanceof TypeVariableNode) { propJ.put("typeLiteral", ((TypeVariableNode)t).getTypeLiteral()); if (!stopRecursive) { - setTypes(propJ, ((TypeVariableNode) t).getRawTypeBound(), typeReplacement, visitedType); + setTypes(propJ, ((TypeVariableNode)t).getRawTypeBound(), typeReplacement, visitedType); } } @@ -1067,7 +1094,7 @@ public class TypeGraph { List<TypeGraphVertex> invalidVertexes = new LinkedList<>(); for (TypeGraphVertex tgv : typeGraph.values()) { - if (tgv.getClassNode() == null) { + if (!tgv.isRealNode()) { invalidVertexes.add(tgv); } } @@ -1126,7 +1153,7 @@ public class TypeGraph private Type resolveType(Type t, Map<Type, Type> typeReplacement) { - if(typeReplacement.containsKey(t)){ + if (typeReplacement.containsKey(t)) { return resolveType(typeReplacement.get(t), typeReplacement); } else { return t; @@ -1203,6 +1230,7 @@ public class TypeGraph TypeGraph result = new TypeGraph(); for (TypeGraphVertex typeGraphVertex : tgv) { result.typeGraph.put(typeGraphVertex.typeName, typeGraphVertex); + typeGraphVertex.setOwner(result); } return result; } @@ -1217,7 +1245,7 @@ public class TypeGraph public List<String> getParents(String className) { TypeGraphVertex tgv = typeGraph.get(className); - if(tgv == null || tgv.ancestors == null){ + if (tgv == null || tgv.ancestors == null) { return null; } List<String> result = new LinkedList<String>(); http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraphFactory.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraphFactory.java b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraphFactory.java index 3be0184..d7cf087 100644 --- a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraphFactory.java +++ b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraphFactory.java @@ -27,14 +27,15 @@ import java.util.Set; import java.util.jar.JarEntry; import java.util.jar.JarFile; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.commons.io.IOUtils; -import com.datatorrent.stram.webapp.TypeGraph.TypeGraphSerializer; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; +import com.datatorrent.stram.webapp.TypeGraph.TypeGraphSerializer; + /** * This class keeps a precomputed index(prototype) of type graph of all classes in jdk and gateway classpath @@ -43,7 +44,7 @@ import com.esotericsoftware.kryo.io.Output; */ public class TypeGraphFactory { - private final static byte[] preComputeGraph; + private static final byte[] preComputeGraph; private static final Logger LOG = LoggerFactory.getLogger(TypeGraphFactory.class); @@ -72,7 +73,7 @@ public class TypeGraphFactory for (String path : pathsToScan) { try { File f = new File(path); - if(!f.exists() || !f.getName().endsWith("jar")){ + if (!f.exists() || !f.getName().endsWith("jar")) { continue; } JarFile jar = new JarFile(path); @@ -101,11 +102,12 @@ public class TypeGraphFactory kryo.writeObject(output, tg); output.close(); preComputeGraph = baos.toByteArray(); - LOG.warn("The size of precomputed type graph is {} KB", preComputeGraph.length/1024); + LOG.warn("The size of precomputed type graph is {} KB", preComputeGraph.length / 1024); } - public static TypeGraph createTypeGraphProtoType(){ + public static TypeGraph createTypeGraphProtoType() + { Input input = null; try { input = new Input(new ByteArrayInputStream(preComputeGraph)); http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/main/java/com/datatorrent/stram/webapp/asm/ASMUtil.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/asm/ASMUtil.java b/engine/src/main/java/com/datatorrent/stram/webapp/asm/ASMUtil.java index e827042..c9b8f15 100644 --- a/engine/src/main/java/com/datatorrent/stram/webapp/asm/ASMUtil.java +++ b/engine/src/main/java/com/datatorrent/stram/webapp/asm/ASMUtil.java @@ -153,6 +153,11 @@ public class ASMUtil return (opCode & Opcodes.ACC_PUBLIC) == Opcodes.ACC_PUBLIC; } + public static boolean isAbstract(int opCode) + { + return (opCode & Opcodes.ACC_ABSTRACT) == Opcodes.ACC_ABSTRACT; + } + public static boolean isTransient(int opCode) { return (opCode & Opcodes.ACC_TRANSIENT) == Opcodes.ACC_TRANSIENT; http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/main/java/com/datatorrent/stram/webapp/asm/ClassNodeType.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/asm/ClassNodeType.java b/engine/src/main/java/com/datatorrent/stram/webapp/asm/ClassNodeType.java index 19e00ac..80f475f 100644 --- a/engine/src/main/java/com/datatorrent/stram/webapp/asm/ClassNodeType.java +++ b/engine/src/main/java/com/datatorrent/stram/webapp/asm/ClassNodeType.java @@ -39,56 +39,40 @@ public class ClassNodeType extends ClassNode } ClassSignatureVisitor csv = new ClassSignatureVisitor(); - private boolean visitFields = false; @SuppressWarnings("unchecked") @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { - if(!isVisitFields()) { MethodNode mn = new MethodNode(access, name, desc, signature, exceptions); mn.typeVariableSignatureNode = csv; methods.add(mn); return mn; - } - return null; } @SuppressWarnings("unchecked") @Override public FieldVisitor visitField(int access, String name, String desc, String signature, Object value) { - if(isVisitFields()) { - FieldNode fn = new FieldNode(access, name, desc, signature, value); - fn.typeVariableSignatureNode = csv; - fields.add(fn); - return fn; - } - return null; + FieldNode fn = new FieldNode(access, name, desc, signature, value); + fn.typeVariableSignatureNode = csv; + fields.add(fn); + return fn; } @Override public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { - if(!isVisitFields()) { - // parse the signature first so Type variable can be captured from the signature - if(signature!=null){ - SignatureReader sr = new SignatureReader(signature); - sr.accept(csv); - } - super.visit(version, access, name, signature, superName, interfaces); + // parse the signature first so Type variable can be captured from the signature + if (signature != null) { + SignatureReader sr = new SignatureReader(signature); + sr.accept(csv); } + super.visit(version, access, name, signature, superName, interfaces); } public void setClassSignatureVisitor(ClassSignatureVisitor csv){ this.csv = csv; } - public boolean isVisitFields() { - return visitFields; - } - - public void setVisitFields(boolean visitFields) { - this.visitFields = visitFields; - } } http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/main/java/com/datatorrent/stram/webapp/asm/CompactUtil.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/asm/CompactUtil.java b/engine/src/main/java/com/datatorrent/stram/webapp/asm/CompactUtil.java index 0b55578..52ab342 100644 --- a/engine/src/main/java/com/datatorrent/stram/webapp/asm/CompactUtil.java +++ b/engine/src/main/java/com/datatorrent/stram/webapp/asm/CompactUtil.java @@ -40,7 +40,7 @@ public class CompactUtil public static CompactClassNode compactClassNode(ClassNode cn) { - if(cn == null){ + if (cn == null) { return null; } CompactClassNode ccn = new CompactClassNode(); @@ -69,11 +69,11 @@ public class CompactUtil inner.setAccess(((InnerClassNode)icn).access); } ccn.setInnerClasses(ccns); - if(ASMUtil.isEnum(cn)){ + if (ASMUtil.isEnum(cn)) { ccn.setEnumValues(ASMUtil.getEnumValues(cn)); } - if(cn instanceof ClassNodeType){ + if (cn instanceof ClassNodeType){ ccn.setCsv(((ClassNodeType)cn).csv); } @@ -97,8 +97,7 @@ public class CompactUtil { List<FieldNode> fields = ASMUtil.getPorts(cn); List<CompactFieldNode> ports = new LinkedList<CompactFieldNode>(); - for(FieldNode fn : fields) - { + for (FieldNode fn : fields) { ports.add(compactFieldNode(fn)); } ccn.setPorts(ports); @@ -124,12 +123,10 @@ public class CompactUtil cfn.setName(fn.name); String className = org.apache.xbean.asm5.Type.getObjectType(fn.desc).getClassName(); - if(className.charAt(0) == 'L') - { + if (className.charAt(0) == 'L') { className = className.substring(1); } - if(className.endsWith(";")) - { + if (className.endsWith(";")) { className = className.substring(0, className.length() - 1); } cfn.setDescription(className); @@ -138,8 +135,9 @@ public class CompactUtil if (fn.visibleAnnotations != null) { setAnnotationNode(fn, cfn); } - if(fn instanceof com.datatorrent.stram.webapp.asm.FieldNode) + if (fn instanceof com.datatorrent.stram.webapp.asm.FieldNode) { cfn.setFieldSignatureNode((((com.datatorrent.stram.webapp.asm.FieldNode)fn).signatureNode)); + } return cfn; } @@ -149,7 +147,7 @@ public class CompactUtil CompactAnnotationNode node = new CompactAnnotationNode(); Map<String, Object> annotationMap = new HashMap<String, Object>(); if (visibleAnnotation instanceof AnnotationNode) { - AnnotationNode annotation = (AnnotationNode) visibleAnnotation; + AnnotationNode annotation = (AnnotationNode)visibleAnnotation; if (annotation.desc.contains("InputPortFieldAnnotation") || annotation.desc.contains("OutputPortFieldAnnotation")) { List<Object> annotationValues = annotation.values; http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/main/java/com/datatorrent/stram/webapp/asm/FastClassIndexReader.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/asm/FastClassIndexReader.java b/engine/src/main/java/com/datatorrent/stram/webapp/asm/FastClassIndexReader.java new file mode 100644 index 0000000..fc689fc --- /dev/null +++ b/engine/src/main/java/com/datatorrent/stram/webapp/asm/FastClassIndexReader.java @@ -0,0 +1,340 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datatorrent.stram.webapp.asm; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; + +import org.apache.xbean.asm5.Opcodes; + +/** + * + * An improvement for ASM class reader. + * ASM class reader reads the whole class into structured data like ClassNode MethodNode + * But to build a type graph we only need to know class name, parent class name, interfaces name and + * whether it has a public non-arg default constructor (instantiable) + * This class skip most parts that are not necessary and parse only the information above + * + * And also it use shared buffer to load classes which makes it faster in our performance test + * + * Overall it is 8-9x faster than ASM class reader + * + * Keep in mind it is NOT thread safe + * + */ +public class FastClassIndexReader +{ + + /** + * The type of CONSTANT_Class constant pool items. + */ + static final int CLASS = 7; + + /** + * The type of CONSTANT_Fieldref constant pool items. + */ + static final int FIELD = 9; + + /** + * The type of CONSTANT_Methodref constant pool items. + */ + static final int METH = 10; + + /** + * The type of CONSTANT_InterfaceMethodref constant pool items. + */ + static final int IMETH = 11; + + /** + * The type of CONSTANT_String constant pool items. + */ + static final int STR = 8; + + /** + * The type of CONSTANT_Integer constant pool items. + */ + static final int INT = 3; + + /** + * The type of CONSTANT_Float constant pool items. + */ + static final int FLOAT = 4; + + /** + * The type of CONSTANT_Long constant pool items. + */ + static final int LONG = 5; + + /** + * The type of CONSTANT_Double constant pool items. + */ + static final int DOUBLE = 6; + + /** + * The type of CONSTANT_NameAndType constant pool items. + */ + static final int NAME_TYPE = 12; + + /** + * The type of CONSTANT_Utf8 constant pool items. + */ + static final int UTF8 = 1; + + /** + * The type of CONSTANT_MethodType constant pool items. + */ + static final int MTYPE = 16; + + /** + * The type of CONSTANT_MethodHandle constant pool items. + */ + static final int HANDLE = 15; + + /** + * The type of CONSTANT_InvokeDynamic constant pool items. + */ + static final int INDY = 18; + + // shared buffer to hold the content of the file + private static byte[] b = new byte[64 * 1024]; + + private static int bSize = 0; + + private int[] items; + + private int header; + + private String name; + + private String superName; + + private String[] interfaces; + + // use this for byte array comparison which is faster than string comparison + public static final byte[] DEFAULT_CONSTRUCTOR_NAME = new byte[]{0, 6, '<', 'i', 'n', 'i', 't', '>'}; + + public static final byte[] DEFAULT_CONSTRUCTOR_DESC = new byte[]{0, 3, '(', ')', 'V'}; + + private boolean isInstantiable = false; + + public FastClassIndexReader(final InputStream is) throws IOException + { + readIntoBuffer(is); + + readConstantPool(); + + readIndex(); + } + + /** + * Read class file content into shared buffer from input stream + * Stream won't be closed + * @param is + * @throws IOException + */ + private void readIntoBuffer(InputStream is) throws IOException + { + if (is == null) { + throw new IOException("Class not found"); + } + bSize = 0; + while (true) { + int n = is.read(b, bSize, b.length - bSize); + if (n == -1) { + break; + } + bSize += n; + if (bSize >= b.length) { + byte[] c = new byte[b.length << 2]; + System.arraycopy(b, 0, c, 0, b.length); + b = c; + } + } + } + + /** + * read and index the constant pool section for getting class metadata later + */ + private void readConstantPool() + { + // checks the class version + if (readShort(6) > Opcodes.V1_8) { + throw new IllegalArgumentException(); + } + // parses the constant pool + items = new int[readUnsignedShort(8)]; + int n = items.length; + int index = 10; + for (int i = 1; i < n; ++i) { + items[i] = index + 1; + int size; + switch (b[index]) { + case FIELD: + case METH: + case IMETH: + case INT: + case FLOAT: + case NAME_TYPE: + case INDY: + size = 5; + break; + case LONG: + case DOUBLE: + size = 9; + ++i; + break; + case UTF8: + size = 3 + readUnsignedShort(index + 1); + break; + case HANDLE: + size = 4; + break; + // case ClassWriter.CLASS: + // case ClassWriter.STR: + // case ClassWriter.MTYPE + default: + size = 3; + break; + } + index += size; + } + // the class header information starts just after the constant pool + header = index; + } + + /** + * read class metadata, class name, parent name, interfaces name, is instantiable(has public non-arg constructor) + * or not + * @throws UnsupportedEncodingException + */ + private void readIndex() throws UnsupportedEncodingException + { + // reads the class declaration + int u = header; + + int access = readUnsignedShort(u); + isInstantiable = ASMUtil.isPublic(access) && !ASMUtil.isAbstract(access); + + name = readClass(u + 2); + superName = readClass(u + 4); + interfaces = new String[readUnsignedShort(u + 6)]; + u += 8; + for (int i = 0; i < interfaces.length; ++i) { + interfaces[i] = readClass(u); + u += 2; + } + + if (!isInstantiable) { + return; + } + + // reads the constructor + + // skip fields + for (int i = readUnsignedShort(u); i > 0; --i) { + for (int j = readUnsignedShort(u + 8); j > 0; --j) { + u += 6 + readInt(u + 12); + } + u += 8; + } + u += 2; + for (int i = readUnsignedShort(u); i > 0; --i) { + if (isDefaultConstructor(u + 2)) { + return; + } + for (int j = readUnsignedShort(u + 8); j > 0; --j) { + u += 6 + readInt(u + 12); + } + u += 8; + } + isInstantiable = false; + } + + private boolean isDefaultConstructor(int methodIndex) throws UnsupportedEncodingException + { + return arrayContains(b, items[readUnsignedShort(methodIndex + 2)], DEFAULT_CONSTRUCTOR_NAME) + && arrayContains(b, items[readUnsignedShort(methodIndex + 4)], DEFAULT_CONSTRUCTOR_DESC) + && ASMUtil.isPublic(readUnsignedShort(methodIndex)); + } + + private boolean arrayContains(byte[] bb, int i, byte[] subArray) + { + for (int l = 0; l < subArray.length; l++) { + if (bb[i + l] != subArray[l]) { + return false; + } + } + return true; + } + + public int readUnsignedShort(final int index) + { + return ((b[index] & 0xFF) << 8) | (b[index + 1] & 0xFF); + } + + public short readShort(final int index) + { + return (short)(((b[index] & 0xFF) << 8) | (b[index + 1] & 0xFF)); + } + + public int readInt(final int index) + { + return ((b[index] & 0xFF) << 24) | ((b[index + 1] & 0xFF) << 16) + | ((b[index + 2] & 0xFF) << 8) | (b[index + 3] & 0xFF); + } + + public String readClass(final int index) throws UnsupportedEncodingException + { + // computes the start index of the CONSTANT_Class item in b + // and reads the CONSTANT_Utf8 item designated by + // the first two bytes of this CONSTANT_Class item + return readUTF8(items[readUnsignedShort(index)]); + } + + public String readUTF8(int index) throws UnsupportedEncodingException + { + int item = readUnsignedShort(index); + if (index == 0 || item == 0) { + return null; + } + index = items[item]; + return new String(b, index + 2, readUnsignedShort(index), "UTF-8"); + } + + public String getName() + { + return name; + } + + public String getSuperName() + { + return superName; + } + + public String[] getInterfaces() + { + return interfaces; + } + + public boolean isInstantiable() + { + return isInstantiable; + } +} http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java ---------------------------------------------------------------------- diff --git a/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java b/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java index 5053c1a..7c1be8f 100644 --- a/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java +++ b/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java @@ -1138,5 +1138,6 @@ public class OperatorDiscoveryTest Assert.assertEquals("@description", OperatorDiscoverer.MethodTagType.DESCRIPTION, OperatorDiscoverer.MethodTagType.from("@description")); Assert.assertEquals("@random", null, OperatorDiscoverer.MethodTagType.from("@random")); } + } http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/engine/src/test/java/com/datatorrent/stram/webapp/asm/FastClassIndexReaderTest.java ---------------------------------------------------------------------- diff --git a/engine/src/test/java/com/datatorrent/stram/webapp/asm/FastClassIndexReaderTest.java b/engine/src/test/java/com/datatorrent/stram/webapp/asm/FastClassIndexReaderTest.java new file mode 100644 index 0000000..41d87bc --- /dev/null +++ b/engine/src/test/java/com/datatorrent/stram/webapp/asm/FastClassIndexReaderTest.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datatorrent.stram.webapp.asm; + +import java.io.IOException; +import java.io.InputStream; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.xbean.asm5.ClassReader; + +public class FastClassIndexReaderTest +{ + private static final Logger LOG = LoggerFactory.getLogger(FastClassIndexReaderTest.class); + + @Test + public void testIndexReader() throws IOException + { + String javahome = System.getProperty("java.home"); + String jdkJar = javahome + "/lib/rt.jar"; + JarFile jar = new JarFile(jdkJar); + java.util.Enumeration<JarEntry> entriesEnum = jar.entries(); + while (entriesEnum.hasMoreElements()) { + JarEntry jarEntry = entriesEnum.nextElement(); + if (jarEntry.getName().endsWith("class")) { + InputStream ins = jar.getInputStream(jarEntry); + ClassReader classReader = new ClassReader(ins); + ClassNodeType classN = new ClassNodeType(); + classReader.accept(classN, ClassReader.SKIP_CODE); + CompactClassNode ccn = CompactUtil.compactClassNode(classN); + ins.close(); + + ins = jar.getInputStream(jarEntry); + FastClassIndexReader fastClassIndexReader = new FastClassIndexReader(ins); + Assert.assertEquals("The wrong class is " + classN.name, classN.name, fastClassIndexReader.getName()); + Assert.assertEquals("The wrong class is " + classN.name, classN.superName, fastClassIndexReader.getSuperName()); + Assert.assertEquals("The wrong class is " + classN.name, + !ASMUtil.isAbstract(classN.access) && ASMUtil.isPublic(classN.access) + && ccn.getDefaultConstructor() != null, fastClassIndexReader.isInstantiable()); + Assert.assertArrayEquals("The wrong class is " + classN.name, + classN.interfaces.toArray(), fastClassIndexReader.getInterfaces()); + + } + } + } + + @Test + public void testPerformance() throws Exception + { + + String javahome = System.getProperty("java.home"); + String jdkJar = javahome + "/lib/rt.jar"; + JarFile jar = new JarFile(jdkJar); + java.util.Enumeration<JarEntry> entriesEnum = jar.entries(); + long time = System.currentTimeMillis(); + while (entriesEnum.hasMoreElements()) { + JarEntry jarEntry = entriesEnum.nextElement(); + if (jarEntry.getName().endsWith("class")) { + InputStream ins = jar.getInputStream(jarEntry); + //FastClassSignatureReader fastClassSignatureReader = new FastClassSignatureReader(ins); + ClassReader classReader = new ClassReader(ins); + ClassNodeType classN = new ClassNodeType(); + classReader.accept(classN, ClassReader.SKIP_CODE); + CompactClassNode ccn = CompactUtil.compactClassNode(classN); + ins.close(); + } + } + + LOG.info("The time to scan jdk using ASM ClassReader {} ", System.currentTimeMillis() - time); + + jar.close(); + + jar = new JarFile("/usr/lib/jvm/java-7-oracle/jre/lib/rt.jar"); + entriesEnum = jar.entries(); + time = System.currentTimeMillis(); + while (entriesEnum.hasMoreElements()) { + JarEntry jarEntry = entriesEnum.nextElement(); + if (jarEntry.getName().endsWith("class")) { + InputStream ins = jar.getInputStream(jarEntry); + FastClassIndexReader fastClassIndexReader = new FastClassIndexReader(ins); + ins.close(); + } + } + + LOG.info("The time to scan jdk using FastClassIndexReader {} ", System.currentTimeMillis() - time); + + } +} http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d14ab1e5/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ff917ff..3197427 100644 --- a/pom.xml +++ b/pom.xml @@ -98,6 +98,7 @@ <email>[email protected]</email> </properties> <excludes> + <exclude>**/*.iml</exclude> <exclude>DISCLAIMER</exclude> <exclude>LICENSE</exclude> <exclude>NOTICE</exclude> @@ -305,13 +306,12 @@ <dependency> <groupId>com.puppycrawl.tools</groupId> <artifactId>checkstyle</artifactId> - <version>6.11.2</version> + <version>6.12.1</version> </dependency> </dependencies> <executions> <execution> <id>checkstyle</id> - <phase>validate</phase> <goals> <goal>check</goal> </goals> @@ -323,6 +323,8 @@ </executions> <configuration> <configLocation>apex_checks.xml</configLocation> + <suppressionsLocation>checkstyle-suppressions.xml</suppressionsLocation> + <suppressionsFileExpression>checkstyle.suppressions.file</suppressionsFileExpression> </configuration> </plugin> <plugin>
