Fix CLassLoading issues and test issues introduced with ANY23-274
Project: http://git-wip-us.apache.org/repos/asf/any23/repo Commit: http://git-wip-us.apache.org/repos/asf/any23/commit/89d1d85e Tree: http://git-wip-us.apache.org/repos/asf/any23/tree/89d1d85e Diff: http://git-wip-us.apache.org/repos/asf/any23/diff/89d1d85e Branch: refs/heads/master Commit: 89d1d85e263ef91f4c696a56a87eb15c2892349b Parents: 1bb96c4 Author: Lewis John McGibbney <[email protected]> Authored: Mon Feb 27 20:56:29 2017 -0800 Committer: Lewis John McGibbney <[email protected]> Committed: Mon Feb 27 20:56:29 2017 -0800 ---------------------------------------------------------------------- .../apache/any23/extractor/ExtractorGroup.java | 14 +++++---- cli/pom.xml | 28 ++++++++++++++---- .../org/apache/any23/cli/PluginVerifier.java | 8 +++--- .../main/java/org/apache/any23/cli/Rover.java | 10 ++++--- .../java/org/apache/any23/cli/ToolRunner.java | 7 +++-- .../any23/extractor/ExtractorRegistryImpl.java | 30 ++------------------ .../any23/extractor/ExtractionAPITest.java | 4 +-- .../extractor/ExtractionResultImplTest.java | 2 +- openie/pom.xml | 10 ------- plugins/basic-crawler/pom.xml | 19 ------------- plugins/html-scraper/pom.xml | 19 ------------- plugins/integration-test/pom.xml | 16 ++++------- .../java/org/apache/any23/plugin/PluginIT.java | 22 +++++++------- plugins/office-scraper/pom.xml | 19 ------------- pom.xml | 18 ++++++++++-- 15 files changed, 83 insertions(+), 143 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/api/src/main/java/org/apache/any23/extractor/ExtractorGroup.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/any23/extractor/ExtractorGroup.java b/api/src/main/java/org/apache/any23/extractor/ExtractorGroup.java index 4eeaa12..9242ea6 100644 --- a/api/src/main/java/org/apache/any23/extractor/ExtractorGroup.java +++ b/api/src/main/java/org/apache/any23/extractor/ExtractorGroup.java @@ -52,7 +52,7 @@ public class ExtractorGroup implements Iterable<ExtractorFactory<?>> { */ public ExtractorGroup filterByMIMEType(MIMEType mimeType) { // @@@ wildcards, q values - Collection<ExtractorFactory<?>> matching = new ArrayList<ExtractorFactory<?>>(); + Collection<ExtractorFactory<?>> matching = new ArrayList<>(); for (ExtractorFactory<?> factory : factories) { if (supportsAllContentTypes(factory) || supports(factory, mimeType)) { matching.add(factory); @@ -71,7 +71,8 @@ public class ExtractorGroup implements Iterable<ExtractorFactory<?>> { */ public boolean allExtractorsSupportAllContentTypes() { for (ExtractorFactory<?> factory : factories) { - if (!supportsAllContentTypes(factory)) return false; + if (!supportsAllContentTypes(factory)) + return false; } return true; } @@ -82,9 +83,12 @@ public class ExtractorGroup implements Iterable<ExtractorFactory<?>> { private boolean supports(ExtractorFactory<?> factory, MIMEType mimeType) { for (MIMEType supported : factory.getSupportedMIMETypes()) { - if (supported.isAnyMajorType()) return true; - if (supported.isAnySubtype() && supported.getMajorType().equals(mimeType.getMajorType())) return true; - if (supported.getFullType().equals(mimeType.getFullType())) return true; + if (supported.isAnyMajorType()) + return true; + if (supported.isAnySubtype() && supported.getMajorType().equals(mimeType.getMajorType())) + return true; + if (supported.getFullType().equals(mimeType.getFullType())) + return true; } return false; } http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/cli/pom.xml ---------------------------------------------------------------------- diff --git a/cli/pom.xml b/cli/pom.xml index 6af107e..3f183ae 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -33,11 +33,6 @@ <dependencies> <dependency> <groupId>${project.groupId}</groupId> - <artifactId>apache-any23-openie</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>${project.groupId}</groupId> <artifactId>apache-any23-api</artifactId> <version>${project.version}</version> </dependency> @@ -76,6 +71,26 @@ <scope>test</scope> </dependency> <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>apache-any23-openie</artifactId> + <version>${project.version}</version> + </dependency> + <!-- dependency> + <groupId>${project.groupId}</groupId> + <artifactId>apache-any23-basic-crawler</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>apache-any23-office-scraper</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>apache-any23-html-scraper</artifactId> + <version>${project.version}</version> + </dependency--> + <dependency> <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </dependency> @@ -217,6 +232,9 @@ <program> <mainClass>org.apache.any23.cli.ToolRunner</mainClass> <name>any23</name> + <jvmSettings> + <maxMemorySize>6000m</maxMemorySize> + </jvmSettings> </program> </programs> <configurationDirectory>conf</configurationDirectory> http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/cli/src/main/java/org/apache/any23/cli/PluginVerifier.java ---------------------------------------------------------------------- diff --git a/cli/src/main/java/org/apache/any23/cli/PluginVerifier.java b/cli/src/main/java/org/apache/any23/cli/PluginVerifier.java index a747b49..f3f3234 100644 --- a/cli/src/main/java/org/apache/any23/cli/PluginVerifier.java +++ b/cli/src/main/java/org/apache/any23/cli/PluginVerifier.java @@ -46,7 +46,7 @@ public class PluginVerifier implements Tool { description = "plugins-dir", converter = FileConverter.class ) - private List<File> pluginsDirs = new LinkedList<File>(); + private List<File> pluginsDirs = new LinkedList<>(); public void run() throws Exception { if (pluginsDirs.isEmpty()) { @@ -78,9 +78,9 @@ public class PluginVerifier implements Tool { private void printPluginData(ExtractorFactory extractorFactory, PrintStream ps) { final Author authorAnnotation = extractorFactory.getClass().getAnnotation(Author.class); - ps.printf("Plugin author : %s\n", authorAnnotation == null ? "<unknown>" : authorAnnotation.name()); - ps.printf("Plugin factory : %s\n", extractorFactory.getClass()); - ps.printf("Plugin mime-types: %s\n", getMimeTypesStr( extractorFactory.getSupportedMIMETypes() )); + ps.printf("Plugin author : %s%n", authorAnnotation == null ? "<unknown>" : authorAnnotation.name()); + ps.printf("Plugin factory : %s%n", extractorFactory.getClass()); + ps.printf("Plugin mime-types: %s%n", getMimeTypesStr( extractorFactory.getSupportedMIMETypes() )); } } http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/cli/src/main/java/org/apache/any23/cli/Rover.java ---------------------------------------------------------------------- diff --git a/cli/src/main/java/org/apache/any23/cli/Rover.java b/cli/src/main/java/org/apache/any23/cli/Rover.java index 26a8663..962e10c 100644 --- a/cli/src/main/java/org/apache/any23/cli/Rover.java +++ b/cli/src/main/java/org/apache/any23/cli/Rover.java @@ -75,10 +75,10 @@ public class Rover implements Tool { private PrintStream outputStream = System.out; @Parameter(description = "input IRIs {<url>|<file>}+", converter = ArgumentToIRIConverter.class) - protected List<String> inputIRIs = new LinkedList<String>(); + protected List<String> inputIRIs = new LinkedList<>(); @Parameter(names = { "-e", "--extractors" }, description = "a comma-separated list of extractors, e.g. rdf-xml,rdf-turtle") - private List<String> extractors = new LinkedList<String>(); + private List<String> extractors = new LinkedList<>(); @Parameter(names = { "-f", "--format" }, description = "the output format") private String format = FORMATS.get(DEFAULT_FORMAT_INDEX); @@ -169,8 +169,10 @@ public class Rover implements Tool { protected String printReports() { final StringBuilder sb = new StringBuilder(); - if (benchmarkTripleHandler != null) sb.append( benchmarkTripleHandler.report() ).append('\n'); - if (reportingTripleHandler != null) sb.append( reportingTripleHandler.printReport() ).append('\n'); + if (benchmarkTripleHandler != null) + sb.append( benchmarkTripleHandler.report() ).append('\n'); + if (reportingTripleHandler != null) + sb.append( reportingTripleHandler.printReport() ).append('\n'); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/cli/src/main/java/org/apache/any23/cli/ToolRunner.java ---------------------------------------------------------------------- diff --git a/cli/src/main/java/org/apache/any23/cli/ToolRunner.java b/cli/src/main/java/org/apache/any23/cli/ToolRunner.java index 90daeb3..b875ec7 100644 --- a/cli/src/main/java/org/apache/any23/cli/ToolRunner.java +++ b/cli/src/main/java/org/apache/any23/cli/ToolRunner.java @@ -223,7 +223,7 @@ public final class ToolRunner { return "z/os"; } else if (osName.contains("os/400")) { return "os/400"; - } else if (pathSep.equals( ";" )) { + } else if (";".equals(pathSep)) { return "dos"; } else if (osName.contains("mac")) { if (osName.endsWith("x")) { @@ -234,7 +234,7 @@ public final class ToolRunner { return "tandem"; } else if (osName.contains("openvms")) { return "openvms"; - } else if (pathSep.equals(":")) { + } else if (":".equals(pathSep)) { return "unix"; } @@ -248,7 +248,8 @@ public final class ToolRunner { optionIndex = i; } } - if(optionIndex == -1) return null; + if(optionIndex == -1) + return null; if(optionIndex == args.length - 1) { throw new IllegalArgumentException("Missing argument for --plugins-dir option."); http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/core/src/main/java/org/apache/any23/extractor/ExtractorRegistryImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/any23/extractor/ExtractorRegistryImpl.java b/core/src/main/java/org/apache/any23/extractor/ExtractorRegistryImpl.java index 736e6bb..86dc982 100644 --- a/core/src/main/java/org/apache/any23/extractor/ExtractorRegistryImpl.java +++ b/core/src/main/java/org/apache/any23/extractor/ExtractorRegistryImpl.java @@ -53,29 +53,6 @@ public class ExtractorRegistryImpl extends org.eclipse.rdf4j.common.lang.service final DefaultConfiguration conf = DefaultConfiguration.singleton(); if (instance == null) { instance = new ExtractorRegistryImpl(); - // FIXME: Remove these hardcoded links to the extractor factories by turning them into SPI interfaces - //instance.register(RDFXMLExtractor.factory); - //instance.register(TurtleExtractor.factory); - //instance.register(NTriplesExtractor.factory); - //instance.register(NQuadsExtractor.factory); - //instance.register(TriXExtractor.factory); - //instance.register(HeadLinkExtractor.factory); - //instance.register(LicenseExtractor.factory); - //instance.register(TitleExtractor.factory); - //instance.register(XFNExtractor.factory); - //instance.register(ICBMExtractor.factory); - //instance.register(AdrExtractor.factory); - //instance.register(GeoExtractor.factory); - //instance.register(HCalendarExtractor.factory); - //instance.register(HCardExtractor.factory); - //instance.register(HListingExtractor.factory); - //instance.register(HResumeExtractor.factory); - //instance.register(HReviewExtractor.factory); - //instance.register(HRecipeExtractor.factory); - //instance.register(SpeciesExtractor.factory); - //instance.register(TurtleHTMLExtractor.factory); - //instance.register(MicrodataExtractor.factory); - //instance.register(CSVExtractor.factory); if(conf.getFlagProperty("any23.extraction.rdfa.programmatic")) { instance.unregister(RDFaExtractorFactory.NAME); @@ -131,8 +108,7 @@ public class ExtractorRegistryImpl extends org.eclipse.rdf4j.common.lang.service */ @Override public ExtractorFactory<?> getFactory(final String name) { - ExtractorFactory<?> result = this.get(name).orElseThrow(() -> new IllegalArgumentException("Unregistered extractor name: " + name)); - return result; + return this.get(name).orElseThrow(() -> new IllegalArgumentException("Unregistered extractor name: " + name)); } /** @@ -152,7 +128,7 @@ public class ExtractorRegistryImpl extends org.eclipse.rdf4j.common.lang.service */ @Override public ExtractorGroup getExtractorGroup(List<String> names) { - List<ExtractorFactory<?>> members = new ArrayList<ExtractorFactory<?>>(names.size()); + List<ExtractorFactory<?>> members = new ArrayList<>(names.size()); for (String name : names) { members.add(getFactory(name)); } @@ -175,7 +151,7 @@ public class ExtractorRegistryImpl extends org.eclipse.rdf4j.common.lang.service */ @Override public List<String> getAllNames() { - List<String> result = new ArrayList<String>(this.getKeys()); + List<String> result = new ArrayList<>(this.getKeys()); Collections.sort(result); return result; } http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/core/src/test/java/org/apache/any23/extractor/ExtractionAPITest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/any23/extractor/ExtractionAPITest.java b/core/src/test/java/org/apache/any23/extractor/ExtractionAPITest.java index abef328..3e1dc73 100644 --- a/core/src/test/java/org/apache/any23/extractor/ExtractionAPITest.java +++ b/core/src/test/java/org/apache/any23/extractor/ExtractionAPITest.java @@ -17,7 +17,7 @@ package org.apache.any23.extractor; -import junit.framework.Assert; +import org.junit.Assert; import org.apache.any23.extractor.example.ExampleExtractor; import org.apache.any23.rdf.RDFUtils; import org.apache.any23.writer.CountingTripleHandler; @@ -30,7 +30,7 @@ import org.eclipse.rdf4j.model.IRI; public class ExtractionAPITest { private static final String exampleDoc = "http://example.com/";; - private static final IRI uri = RDFUtils.iri(exampleDoc); + private static final IRI uri = RDFUtils.iri(exampleDoc); @Test public void testDirectInstantiation() throws Exception { http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/core/src/test/java/org/apache/any23/extractor/ExtractionResultImplTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/any23/extractor/ExtractionResultImplTest.java b/core/src/test/java/org/apache/any23/extractor/ExtractionResultImplTest.java index e21be18..69d1c41 100644 --- a/core/src/test/java/org/apache/any23/extractor/ExtractionResultImplTest.java +++ b/core/src/test/java/org/apache/any23/extractor/ExtractionResultImplTest.java @@ -17,7 +17,7 @@ package org.apache.any23.extractor; -import junit.framework.Assert; +import org.junit.Assert; import org.apache.any23.extractor.html.TitleExtractor; import org.apache.any23.rdf.RDFUtils; import org.apache.any23.writer.TripleHandler; http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/openie/pom.xml ---------------------------------------------------------------------- diff --git a/openie/pom.xml b/openie/pom.xml index 9745b7a..32d4a0c 100644 --- a/openie/pom.xml +++ b/openie/pom.xml @@ -97,16 +97,6 @@ </dependencies> <build> - <resources> - <resource> - <directory>${basedir}/../</directory> - <targetPath>META-INF</targetPath> - <includes> - <include>LICENSE.txt</include> - <include>NOTICE.txt</include> - </includes> - </resource> - </resources> <pluginManagement> <plugins> <plugin> http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/plugins/basic-crawler/pom.xml ---------------------------------------------------------------------- diff --git a/plugins/basic-crawler/pom.xml b/plugins/basic-crawler/pom.xml index 9f294ff..c9769fd 100644 --- a/plugins/basic-crawler/pom.xml +++ b/plugins/basic-crawler/pom.xml @@ -94,15 +94,6 @@ <scope>provided</scope> </dependency> - <!-- BEGIN: plugins --> - <dependency> - <groupId>org.kohsuke.metainf-services</groupId> - <artifactId>metainf-services</artifactId> - <scope>compile</scope> - <optional>true</optional> - </dependency> - <!-- END: plugins --> - <!-- BEGIN: Test Dependencies --> <dependency> <groupId>junit</groupId> @@ -114,16 +105,6 @@ </dependencies> <build> - <resources> - <resource> - <directory>${basedir}/../../</directory> - <targetPath>META-INF</targetPath> - <includes> - <include>LICENSE.txt</include> - <include>NOTICE.txt</include> - </includes> - </resource> - </resources> <plugins> <plugin> http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/plugins/html-scraper/pom.xml ---------------------------------------------------------------------- diff --git a/plugins/html-scraper/pom.xml b/plugins/html-scraper/pom.xml index aec0259..108b675 100644 --- a/plugins/html-scraper/pom.xml +++ b/plugins/html-scraper/pom.xml @@ -71,15 +71,6 @@ </dependency> <!-- END: BoilerPipe --> - <!-- BEGIN: plugins --> - <dependency> - <groupId>org.kohsuke.metainf-services</groupId> - <artifactId>metainf-services</artifactId> - <scope>compile</scope> - <optional>true</optional> - </dependency> - <!-- END: plugins --> - <!-- Logging --> <dependency> <groupId>org.slf4j</groupId> @@ -104,16 +95,6 @@ </dependencies> <build> - <resources> - <resource> - <directory>${basedir}/../../</directory> - <targetPath>META-INF</targetPath> - <includes> - <include>LICENSE.txt</include> - <include>NOTICE.txt</include> - </includes> - </resource> - </resources> <plugins> <!-- Generates the distribution package --> http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/plugins/integration-test/pom.xml ---------------------------------------------------------------------- diff --git a/plugins/integration-test/pom.xml b/plugins/integration-test/pom.xml index d8d87e7..8e8ac9c 100644 --- a/plugins/integration-test/pom.xml +++ b/plugins/integration-test/pom.xml @@ -38,6 +38,11 @@ <version>${project.parent.version}</version> </dependency> <dependency> + <groupId>org.apache.any23</groupId> + <artifactId>apache-any23-cli</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> <groupId>org.apache.any23.plugins</groupId> <artifactId>apache-any23-html-scraper</artifactId> <version>1.0.7-SNAPSHOT</version> @@ -69,16 +74,6 @@ </dependencies> <build> - <resources> - <resource> - <directory>${basedir}/../../</directory> - <targetPath>META-INF</targetPath> - <includes> - <include>LICENSE.txt</include> - <include>NOTICE.txt</include> - </includes> - </resource> - </resources> <plugins> <!-- skip assembly generation for ITs --> @@ -98,7 +93,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-invoker-plugin</artifactId> - <version>1.5</version> <configuration> <projectsDirectory>..</projectsDirectory> <cloneProjectsTo>${project.build.directory}/plugins-build</cloneProjectsTo> http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/plugins/integration-test/src/test/java/org/apache/any23/plugin/PluginIT.java ---------------------------------------------------------------------- diff --git a/plugins/integration-test/src/test/java/org/apache/any23/plugin/PluginIT.java b/plugins/integration-test/src/test/java/org/apache/any23/plugin/PluginIT.java index ce74971..4347c8b 100644 --- a/plugins/integration-test/src/test/java/org/apache/any23/plugin/PluginIT.java +++ b/plugins/integration-test/src/test/java/org/apache/any23/plugin/PluginIT.java @@ -17,6 +17,7 @@ package org.apache.any23.plugin; +import org.apache.any23.cli.Crawler; import org.apache.any23.cli.Tool; import org.apache.any23.extractor.ExtractorGroup; import org.apache.any23.extractor.ExtractorRegistryImpl; @@ -40,8 +41,7 @@ import static org.junit.Assert.assertTrue; */ public class PluginIT { - //TODO reduced from 31 to 28 within ANY23-276 - private static final int NUM_OF_EXTRACTORS = 29; + private static final int NUM_OF_EXTRACTORS = 32; private static final String PLUGIN_DIR = "target/plugins-build/"; @@ -101,16 +101,14 @@ public class PluginIT { tool = tools.next(); assertTrue("Found duplicate tool.", toolClasses.add(tool.getClass().getName())); } -//TODO Crawler.class not on classpath due to ANY23-276 -// assertTrue( -// String.format( -// "Expected [%s] plugin to be detected, but it is not found in the built classpath.", -// Crawler.class.getName() -// ), -// toolClasses.contains(Crawler.class.getName()) -// ); - //TODO Crawler.class not on classpath due to ANY23-276, should be 7 detected CLI including CrawlerCLI - assertEquals(6, toolClasses.size()); // core CLIs + assertTrue( + String.format( + "Expected [%s] plugin to be detected, but it is not found in the built classpath.", + Crawler.class.getName() + ), + toolClasses.contains(Crawler.class.getName()) + ); + assertEquals(7, toolClasses.size()); // core CLIs } } http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/plugins/office-scraper/pom.xml ---------------------------------------------------------------------- diff --git a/plugins/office-scraper/pom.xml b/plugins/office-scraper/pom.xml index ebaa599..6d7e7b8 100644 --- a/plugins/office-scraper/pom.xml +++ b/plugins/office-scraper/pom.xml @@ -61,15 +61,6 @@ <scope>test</scope> </dependency> - <!-- BEGIN: plugins --> - <dependency> - <groupId>org.kohsuke.metainf-services</groupId> - <artifactId>metainf-services</artifactId> - <scope>compile</scope> - <optional>true</optional> - </dependency> - <!-- END: plugins --> - <!-- BEGIN: Test Dependencies --> <dependency> <groupId>junit</groupId> @@ -86,16 +77,6 @@ </dependencies> <build> - <resources> - <resource> - <directory>${basedir}/../../</directory> - <targetPath>META-INF</targetPath> - <includes> - <include>LICENSE.txt</include> - <include>NOTICE.txt</include> - </includes> - </resource> - </resources> <plugins> <!-- Generates the distribution package --> http://git-wip-us.apache.org/repos/asf/any23/blob/89d1d85e/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index fffc7b5..168232e 100644 --- a/pom.xml +++ b/pom.xml @@ -555,8 +555,22 @@ <repository> <id>sonatype-snapshots</id> <url>https://oss.sonatype.org/content/repositories/snapshots/</url> - <releases><enabled>false</enabled></releases> - <snapshots><enabled>true</enabled></snapshots> + <releases> + <enabled>false</enabled> + </releases> + <snapshots> + <enabled>true</enabled> + </snapshots> + </repository> + <repository> + <id>apache-snapshots</id> + <url>https://repository.apache.org/content/repositories/snapshots/</url> + <releases> + <enabled>false</enabled> + </releases> + <snapshots> + <enabled>true</enabled> + </snapshots> </repository> </repositories>
