Repository: incubator-apex-core Updated Branches: refs/heads/devel-3 d748ed46f -> f2a40717b
APEX-70 #resolve #comment add hasResource label to types in typegraph Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/00b486f0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/00b486f0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/00b486f0 Branch: refs/heads/devel-3 Commit: 00b486f0040294870df3d42a88b6fd0ce143d520 Parents: f28d8d7 Author: siyuan <[email protected]> Authored: Mon Aug 31 17:25:22 2015 -0700 Committer: siyuan <[email protected]> Committed: Mon Aug 31 17:25:22 2015 -0700 ---------------------------------------------------------------------- .../stram/webapp/OperatorDiscoverer.java | 44 ++++++++++++++++---- .../com/datatorrent/stram/webapp/TypeGraph.java | 39 +++++++++++++---- .../stram/webapp/OperatorDiscoveryTest.java | 21 ++++++++++ .../example/mydtapp/StdoutOperator/ExtRes.html | 1 + 4 files changed, 90 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/00b486f0/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java b/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java index b35efe6..f08a96b 100644 --- a/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java +++ b/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java @@ -25,6 +25,7 @@ import com.datatorrent.stram.webapp.asm.CompactFieldNode; import com.google.common.base.Predicate; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -277,6 +278,8 @@ public class OperatorDiscoverer { Map<String, JarFile> openJarFiles = new HashMap<String, JarFile>(); Map<String, File> openClassFiles = new HashMap<String, File>(); + // use global cache to load resource in/out of the same jar as the classes + Set<String> resourceCacheSet = new HashSet<>(); try { for (String path : pathsToScan) { File f = null; @@ -296,16 +299,44 @@ public class OperatorDiscoverer openJarFiles.put(path, jar); java.util.Enumeration<JarEntry> entriesEnum = jar.entries(); while (entriesEnum.hasMoreElements()) { - java.util.jar.JarEntry jarEntry = entriesEnum.nextElement(); - if (!jarEntry.isDirectory() && jarEntry.getName().endsWith("-javadoc.xml")) { + final java.util.jar.JarEntry jarEntry = entriesEnum.nextElement(); + String entryName = jarEntry.getName(); + if (jarEntry.isDirectory()) { + continue; + } + if (entryName.endsWith("-javadoc.xml")) { try { processJavadocXml(jar.getInputStream(jarEntry)); // break; } catch (Exception ex) { - LOG.warn("Cannot process javadoc {} : ", jarEntry.getName(), ex); + LOG.warn("Cannot process javadoc {} : ", entryName, ex); + } + } else if (entryName.endsWith(".class")) { + TypeGraph.TypeGraphVertex newNode = typeGraph.addNode(jarEntry, jar); + // check if any visited resources belong to this type + for (Iterator<String> iter = resourceCacheSet.iterator(); iter.hasNext(); ) { + String entry = iter.next(); + if (entry.startsWith(entryName.substring(0, entryName.length() - 6))) { + newNode.setHasResource(true); + iter.remove(); + } + } + } else { + String className = entryName; + boolean foundClass = false; + // check if this resource belongs to any visited type + while (className.contains("/")) { + className = className.substring(0, className.lastIndexOf('/')); + TypeGraph.TypeGraphVertex tgv = typeGraph.getNode(className.replace('/', '.')); + if (tgv != null) { + tgv.setHasResource(true); + foundClass = true; + break; + } + } + if (!foundClass) { + resourceCacheSet.add(entryName); } - } else if (!jarEntry.isDirectory() && jarEntry.getName().endsWith(".class")) { - typeGraph.addNode(jarEntry, jar); } } } @@ -317,8 +348,7 @@ public class OperatorDiscoverer typeGraph.trim(); typeGraph.updatePortTypeInfoInTypeGraph(openJarFiles, openClassFiles); - } - finally { + } finally { for (Entry<String, JarFile> entry : openJarFiles.entrySet()) { try { entry.getValue().close(); http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/00b486f0/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java index fc175af..95d7529 100644 --- a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java +++ b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java @@ -125,7 +125,12 @@ public class TypeGraph } return false; } - + + public TypeGraphVertex getNode(String typeName) + { + return typeGraph.get(typeName); + } + enum UI_TYPE { LIST("List", Collection.class.getName()), @@ -234,7 +239,7 @@ public class TypeGraph private final Map<String, TypeGraphVertex> typeGraph = new HashMap<String, TypeGraphVertex>(); - private void addNode(InputStream input, String resName) throws IOException + private TypeGraphVertex addNode(InputStream input, String resName) throws IOException { try { @@ -243,7 +248,6 @@ public class TypeGraph reader.accept(classN, ClassReader.SKIP_CODE); CompactClassNode ccn = CompactUtil.compactClassNode(classN); String typeName = classN.name.replace('/', '.'); - TypeGraphVertex tgv = null; TypeGraphVertex ptgv = null; if (typeGraph.containsKey(typeName)) { @@ -279,6 +283,7 @@ public class TypeGraph } updateInstantiableDescendants(tgv); + return tgv; } finally { if (input != null) { input.close(); @@ -286,14 +291,14 @@ public class TypeGraph } } - public void addNode(File file) throws IOException + public TypeGraphVertex addNode(File file) throws IOException { - addNode(new FileInputStream(file), file.getAbsolutePath()); + return addNode(new FileInputStream(file), file.getAbsolutePath()); } - public void addNode(JarEntry jarEntry, JarFile jar) throws IOException + public TypeGraphVertex addNode(JarEntry jarEntry, JarFile jar) throws IOException { - addNode(jar.getInputStream(jarEntry), jar.getName()); + return addNode(jar.getInputStream(jarEntry), jar.getName()); } public void updatePortTypeInfoInTypeGraph(Map<String, JarFile> openJarFiles, @@ -477,7 +482,9 @@ public class TypeGraph // keep the jar file name for late fetching the detail information private String jarName; - + + private boolean hasResource = false; + @SuppressWarnings("unused") private TypeGraphVertex(){ jarName = ""; @@ -507,6 +514,16 @@ public class TypeGraph this.jarName = jarName; } + public boolean hasResource() + { + return hasResource; + } + + public void setHasResource(boolean hasResource) + { + this.hasResource = hasResource; + } + public boolean isInstantiable() { return JACKSON_INSTANTIABLE_CLASSES.contains(this.typeName) || (isPublicConcrete() && classNode.getDefaultConstructor() != null); @@ -647,6 +664,12 @@ public class TypeGraph addClassPropertiesAndPorts(clazzName, desc); + if(tgv.hasResource()){ + desc.put("hasResource", "true"); + } else { + desc.put("hasResource", "false"); + } + return desc; } http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/00b486f0/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java ---------------------------------------------------------------------- diff --git a/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java b/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java index 6ac3438..9af7d34 100644 --- a/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java +++ b/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java @@ -49,6 +49,7 @@ import com.datatorrent.api.InputOperator; import com.datatorrent.stram.plan.logical.LogicalPlan; import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta; import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration; +import com.datatorrent.stram.support.StramTestSupport; import com.datatorrent.stram.util.ObjectMapperFactory; import com.datatorrent.stram.webapp.TypeDiscoverer.UI_TYPE; @@ -494,6 +495,26 @@ public class OperatorDiscoveryTest } + @Test + public void testExternalResource() throws Exception + { + + + StramTestSupport.createAppPackageFile(); + + String[] classFilePath = getClassFileInClasspath(); + + String[]cPaths = Lists.asList("src/test/resources/testAppPackage/mydtapp/target/mydtapp-1.0-SNAPSHOT.jar", classFilePath).toArray(new String[]{}); + OperatorDiscoverer od = new OperatorDiscoverer(cPaths); + od.buildTypeGraph(); + + Assert.assertEquals("true", od.describeClass("com.example.mydtapp.StdoutOperator").getString("hasResource")); + + StramTestSupport.removeAppPackageFile(); + + } + + public static class Structured { private int size; http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/00b486f0/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/com/example/mydtapp/StdoutOperator/ExtRes.html ---------------------------------------------------------------------- diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/com/example/mydtapp/StdoutOperator/ExtRes.html b/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/com/example/mydtapp/StdoutOperator/ExtRes.html new file mode 100644 index 0000000..5b23309 --- /dev/null +++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/com/example/mydtapp/StdoutOperator/ExtRes.html @@ -0,0 +1 @@ +<html>example</html>
