Author: tobr
Date: Mon Sep 10 13:23:35 2012
New Revision: 1382824
URL: http://svn.apache.org/viewvc?rev=1382824&view=rev
Log:
changed formatting
Modified:
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/ExceptionCrawlingDroid.java
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/FileRenameDroid.java
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/SaveCrawlingDroid.java
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/SysoutCrawlingDroid.java
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/handler/ExceptionReportHandler.java
incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java
incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestSimpleDroid.java
Modified:
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java?rev=1382824&r1=1382823&r2=1382824&view=diff
==============================================================================
---
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java
(original)
+++
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java
Mon Sep 10 13:23:35 2012
@@ -1,21 +1,21 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.droids.examples;
import java.util.ArrayList;
@@ -41,128 +41,128 @@ import org.apache.droids.robot.crawler.C
import org.apache.droids.robot.crawler.ReportCrawlingDroid;
import org.apache.droids.tika.TikaDocumentParser;
-
/**
*
* Helper class for creating defaults.
- *
+ *
*/
-public class DroidsFactory
-{
-
- public static ParserFactory createDefaultParserFactory() {
- ParserFactory parserFactory = new ParserFactory();
- TikaDocumentParser tikaParser = new TikaDocumentParser();
- parserFactory.getMap().put("text/html", tikaParser);
- return parserFactory;
- }
-
- public static ProtocolFactory createDefaultProtocolFactory() {
- ProtocolFactory protocolFactory = new ProtocolFactory();
- HttpProtocol httpProtocol = new HttpProtocol();
- httpProtocol.setForceAllow(true);
-
- protocolFactory.getMap().put("http", httpProtocol);
- return protocolFactory;
- }
-
- public static URLFiltersFactory createDefaultURLFiltersFactory() {
- URLFiltersFactory filtersFactory = new URLFiltersFactory();
- URLFilter defaultURLFilter = new URLFilter() {
-
- public String filter(String urlString) {
- return urlString;
- }
-
- };
- filtersFactory.getMap().put("default", defaultURLFilter);
- return filtersFactory;
- }
-
- public static HandlerFactory createDefaultHandlerFactory(Handler
defaultHandler) {
- HandlerFactory handlerFactory = new HandlerFactory();
- handlerFactory.getMap().put("default", defaultHandler);
- return handlerFactory;
- }
-
- public static Droid<Link> createSimpleSaveCrawlingDroid(
- String targetURI) {
- ParserFactory parserFactory = createDefaultParserFactory();
- ProtocolFactory protocolFactory = createDefaultProtocolFactory();
- URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
-
- SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
- simpleDelayTimer.setDelayMillis(100);
-
- SimpleTaskQueueWithHistory<Link> simpleQueue = new
SimpleTaskQueueWithHistory<Link>();
-
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
- taskMaster.setDelayTimer( simpleDelayTimer );
- taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
-
- CrawlingDroid crawler = new SaveCrawlingDroid( simpleQueue, taskMaster,
new SaveHandler() );
- crawler.setFiltersFactory(filtersFactory);
- crawler.setParserFactory(parserFactory);
- crawler.setProtocolFactory(protocolFactory);
-
- Collection<String> initialLocations = new ArrayList<String>();
- initialLocations.add( targetURI );
- crawler.setInitialLocations(initialLocations);
- return crawler;
- }
-
- public static Droid<Link> createSimpleReportCrawlingDroid(
- String targetURI) {
- ParserFactory parserFactory = createDefaultParserFactory();
- ProtocolFactory protocolFactory = createDefaultProtocolFactory();
- URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
-
- SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
- simpleDelayTimer.setDelayMillis(100);
-
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
-// MultiThreadedTaskMaster<Link> taskMaster = new
MultiThreadedTaskMaster<Link>();
- taskMaster.setDelayTimer( simpleDelayTimer );
- taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
-
- Queue<Link> queue = new LinkedList<Link>();
-
- CrawlingDroid crawler = new ReportCrawlingDroid( queue, taskMaster );
- crawler.setFiltersFactory(filtersFactory);
- crawler.setParserFactory(parserFactory);
- crawler.setProtocolFactory(protocolFactory);
-
- Collection<String> initialLocations = new ArrayList<String>();
- initialLocations.add( targetURI );
- crawler.setInitialLocations(initialLocations);
- return crawler;
- }
-
- public static Droid<Link> createSimpleExceptionCrawlingDroid(
- String targetURI) {
- ParserFactory parserFactory = createDefaultParserFactory();
- ProtocolFactory protocolFactory = createDefaultProtocolFactory();
- URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
-
- SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
- simpleDelayTimer.setDelayMillis(100);
-
- Queue<Link> queue = new LinkedList<Link>();
-
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
-// MultiThreadedTaskMaster<Link> taskMaster = new
MultiThreadedTaskMaster<Link>();
- taskMaster.setDelayTimer( simpleDelayTimer );
- taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
-
- CrawlingDroid crawler = new ExceptionCrawlingDroid( queue, taskMaster );
- crawler.setFiltersFactory(filtersFactory);
- crawler.setParserFactory(parserFactory);
- crawler.setProtocolFactory(protocolFactory);
-
- Collection<String> initialLocations = new ArrayList<String>();
- initialLocations.add( targetURI );
- crawler.setInitialLocations(initialLocations);
- return crawler;
- }
-
+public class DroidsFactory {
+
+ public static ParserFactory createDefaultParserFactory() {
+ ParserFactory parserFactory = new ParserFactory();
+ TikaDocumentParser tikaParser = new TikaDocumentParser();
+ parserFactory.getMap().put("text/html", tikaParser);
+ return parserFactory;
+ }
+
+ public static ProtocolFactory createDefaultProtocolFactory() {
+ ProtocolFactory protocolFactory = new ProtocolFactory();
+ HttpProtocol httpProtocol = new HttpProtocol();
+ httpProtocol.setForceAllow(true);
+
+ protocolFactory.getMap().put("http", httpProtocol);
+ return protocolFactory;
+ }
+
+ public static URLFiltersFactory createDefaultURLFiltersFactory() {
+ URLFiltersFactory filtersFactory = new URLFiltersFactory();
+ URLFilter defaultURLFilter = new URLFilter() {
+
+ public String filter(String urlString) {
+ return urlString;
+ }
+
+ };
+ filtersFactory.getMap().put("default", defaultURLFilter);
+ return filtersFactory;
+ }
+
+ public static HandlerFactory createDefaultHandlerFactory(
+ Handler defaultHandler) {
+ HandlerFactory handlerFactory = new HandlerFactory();
+ handlerFactory.getMap().put("default", defaultHandler);
+ return handlerFactory;
+ }
+
+ public static Droid<Link> createSimpleSaveCrawlingDroid(String
targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory =
createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory =
createDefaultURLFiltersFactory();
+
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+ SimpleTaskQueueWithHistory<Link> simpleQueue = new
SimpleTaskQueueWithHistory<Link>();
+
+ SequentialTaskMaster<Link> taskMaster = new
SequentialTaskMaster<Link>();
+ taskMaster.setDelayTimer(simpleDelayTimer);
+ taskMaster.setExceptionHandler(new
DefaultTaskExceptionHandler());
+
+ CrawlingDroid crawler = new SaveCrawlingDroid(simpleQueue,
taskMaster,
+ new SaveHandler());
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add(targetURI);
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
+ public static Droid<Link> createSimpleReportCrawlingDroid(String
targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory =
createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory =
createDefaultURLFiltersFactory();
+
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+ SequentialTaskMaster<Link> taskMaster = new
SequentialTaskMaster<Link>();
+ // MultiThreadedTaskMaster<Link> taskMaster = new
+ // MultiThreadedTaskMaster<Link>();
+ taskMaster.setDelayTimer(simpleDelayTimer);
+ taskMaster.setExceptionHandler(new
DefaultTaskExceptionHandler());
+
+ Queue<Link> queue = new LinkedList<Link>();
+
+ CrawlingDroid crawler = new ReportCrawlingDroid(queue,
taskMaster);
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add(targetURI);
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
+ public static Droid<Link> createSimpleExceptionCrawlingDroid(
+ String targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory =
createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory =
createDefaultURLFiltersFactory();
+
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+ Queue<Link> queue = new LinkedList<Link>();
+
+ SequentialTaskMaster<Link> taskMaster = new
SequentialTaskMaster<Link>();
+ // MultiThreadedTaskMaster<Link> taskMaster = new
+ // MultiThreadedTaskMaster<Link>();
+ taskMaster.setDelayTimer(simpleDelayTimer);
+ taskMaster.setExceptionHandler(new
DefaultTaskExceptionHandler());
+
+ CrawlingDroid crawler = new ExceptionCrawlingDroid(queue,
taskMaster);
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add(targetURI);
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
}
Modified:
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/ExceptionCrawlingDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/ExceptionCrawlingDroid.java?rev=1382824&r1=1382823&r2=1382824&view=diff
==============================================================================
---
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/ExceptionCrawlingDroid.java
(original)
+++
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/ExceptionCrawlingDroid.java
Mon Sep 10 13:23:35 2012
@@ -1,21 +1,21 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.droids.examples;
import java.util.Queue;
@@ -29,16 +29,17 @@ import org.apache.droids.robot.crawler.C
public class ExceptionCrawlingDroid extends CrawlingDroid {
- public ExceptionCrawlingDroid(Queue<Link> queue, TaskMaster<Link>
taskMaster) {
- super(queue, taskMaster);
- }
+ public ExceptionCrawlingDroid(Queue<Link> queue, TaskMaster<Link>
taskMaster) {
+ super(queue, taskMaster);
+ }
- @Override
- public Worker<Link> getNewWorker() {
- final CrawlingWorker worker = new CrawlingWorker( this );
- Handler testHandler = new ExceptionReportHandler();
-
worker.setHandlerFactory(DroidsFactory.createDefaultHandlerFactory(testHandler));
- return worker;
- }
+ @Override
+ public Worker<Link> getNewWorker() {
+ final CrawlingWorker worker = new CrawlingWorker(this);
+ Handler testHandler = new ExceptionReportHandler();
+ worker.setHandlerFactory(DroidsFactory
+ .createDefaultHandlerFactory(testHandler));
+ return worker;
+ }
}
Modified:
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/FileRenameDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/FileRenameDroid.java?rev=1382824&r1=1382823&r2=1382824&view=diff
==============================================================================
---
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/FileRenameDroid.java
(original)
+++
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/FileRenameDroid.java
Mon Sep 10 13:23:35 2012
@@ -32,113 +32,104 @@ import org.apache.droids.impl.MultiThrea
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class FileRenameDroid extends AbstractDroid<FileTask>
-{
+public class FileRenameDroid extends AbstractDroid<FileTask> {
- private static final Logger LOG =
LoggerFactory.getLogger(FileRenameDroid.class);
- private Collection<File> initialFiles;
-
- public FileRenameDroid(Queue<FileTask> queue, TaskMaster<FileTask>
taskMaster)
- {
- super(queue, taskMaster);
- }
-
- public void setInitialFiles(Collection<File> initialFiles)
- {
- this.initialFiles = initialFiles;
- }
-
- public void init()
- {
- Preconditions.checkNotNull(initialFiles);
- Preconditions.checkState(!initialFiles.isEmpty());
- for (File file : initialFiles) {
- queue.add(new FileTask(file, 0));
- }
- }
- public LinkedHashMap<String, String> cleaner = null;
-
- public LinkedHashMap<String, String> getCleaner()
- {
- if (null == cleaner) {
- populateCleaner();
- }
- return cleaner;
- }
-
- public void setCleaner(LinkedHashMap<String, String> cleaner)
- {
- this.cleaner = cleaner;
- }
-
- private void populateCleaner()
- {
- cleaner = new LinkedHashMap<String, String>();
- cleaner.put(" ", ".");
- cleaner.put(".-.", ".");
- cleaner.put(",", "");
- }
-
- public void finished()
- {
- System.out.println("FINISHED!!!");
- }
-
- public RenameWorker getNewWorker()
- {
- return new RenameWorker();
- }
-
- public class RenameWorker implements Worker<FileTask>
- {
-
- String replace;
-
- public void execute(FileTask task)
- {
-
- for (String pattern : getCleaner().keySet()) {
- replace = getCleaner().get(pattern);
-
- cleanFileName(task.getFile(), pattern, replace);
- }
- }
-
- private void cleanFileName(File file, String pattern, String replace)
- {
- LOG.debug("Processing: " + file.getName());
- LOG.debug("finding pattern: " + pattern);
- LOG.debug("replacing it with: " + replace);
- String fileName = file.getName();
- if (fileName.contains(pattern) ||
!fileName.toLowerCase().equals(fileName)) {
- LOG.debug("need to process this file: " + fileName + " in "
- + file.getAbsolutePath());
- File replacement = new File(fileName.substring(0,
file.getAbsolutePath().indexOf(fileName))
- + fileName.replaceAll(pattern, replace).toLowerCase());
- LOG.debug("Renaming to: " + replacement.getName() + " in "
- + replacement.getAbsolutePath());
-
- LOG.info("TODO! actually do the rename!");
- // TODO -- actually do the rename...file.renameTo(replacement);
- }
- }
- }
-
- //------------------------------------------------------------------
- //------------------------------------------------------------------
- public static void main(String[] args)
- {
- MultiThreadedTaskMaster<FileTask> taskMaster = new
MultiThreadedTaskMaster<FileTask>();
- taskMaster.setPoolSize(3);
-
- Queue<FileTask> queue = new LinkedList<FileTask>();
-
- Collection<File> files = new ArrayList<File>();
- files.add(new File(args[0]));
-
- FileRenameDroid simple = new FileRenameDroid(queue, taskMaster);
- simple.setInitialFiles(files);
- simple.init();
- simple.start(); // TODO? perhaps start internally calls init()?
- }
+ private static final Logger LOG = LoggerFactory
+ .getLogger(FileRenameDroid.class);
+ private Collection<File> initialFiles;
+
+ public FileRenameDroid(Queue<FileTask> queue, TaskMaster<FileTask>
taskMaster) {
+ super(queue, taskMaster);
+ }
+
+ public void setInitialFiles(Collection<File> initialFiles) {
+ this.initialFiles = initialFiles;
+ }
+
+ public void init() {
+ Preconditions.checkNotNull(initialFiles);
+ Preconditions.checkState(!initialFiles.isEmpty());
+ for (File file : initialFiles) {
+ queue.add(new FileTask(file, 0));
+ }
+ }
+
+ public LinkedHashMap<String, String> cleaner = null;
+
+ public LinkedHashMap<String, String> getCleaner() {
+ if (null == cleaner) {
+ populateCleaner();
+ }
+ return cleaner;
+ }
+
+ public void setCleaner(LinkedHashMap<String, String> cleaner) {
+ this.cleaner = cleaner;
+ }
+
+ private void populateCleaner() {
+ cleaner = new LinkedHashMap<String, String>();
+ cleaner.put(" ", ".");
+ cleaner.put(".-.", ".");
+ cleaner.put(",", "");
+ }
+
+ public void finished() {
+ System.out.println("FINISHED!!!");
+ }
+
+ public RenameWorker getNewWorker() {
+ return new RenameWorker();
+ }
+
+ public class RenameWorker implements Worker<FileTask> {
+
+ String replace;
+
+ public void execute(FileTask task) {
+
+ for (String pattern : getCleaner().keySet()) {
+ replace = getCleaner().get(pattern);
+
+ cleanFileName(task.getFile(), pattern, replace);
+ }
+ }
+
+ private void cleanFileName(File file, String pattern, String
replace) {
+ LOG.debug("Processing: " + file.getName());
+ LOG.debug("finding pattern: " + pattern);
+ LOG.debug("replacing it with: " + replace);
+ String fileName = file.getName();
+ if (fileName.contains(pattern)
+ ||
!fileName.toLowerCase().equals(fileName)) {
+ LOG.debug("need to process this file: " +
fileName + " in "
+ + file.getAbsolutePath());
+ File replacement = new
File(fileName.substring(0, file
+
.getAbsolutePath().indexOf(fileName))
+ + fileName.replaceAll(pattern,
replace).toLowerCase());
+ LOG.debug("Renaming to: " +
replacement.getName() + " in "
+ +
replacement.getAbsolutePath());
+
+ LOG.info("TODO! actually do the rename!");
+ // TODO -- actually do the
rename...file.renameTo(replacement);
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------
+ // ------------------------------------------------------------------
+ public static void main(String[] args) {
+ MultiThreadedTaskMaster<FileTask> taskMaster = new
MultiThreadedTaskMaster<FileTask>();
+ taskMaster.setPoolSize(3);
+
+ Queue<FileTask> queue = new LinkedList<FileTask>();
+
+ Collection<File> files = new ArrayList<File>();
+ files.add(new File(args[0]));
+
+ FileRenameDroid simple = new FileRenameDroid(queue, taskMaster);
+ simple.setInitialFiles(files);
+ simple.init();
+ simple.start(); // TODO? perhaps start internally calls init()?
+ }
}
Modified:
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/SaveCrawlingDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/SaveCrawlingDroid.java?rev=1382824&r1=1382823&r2=1382824&view=diff
==============================================================================
---
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/SaveCrawlingDroid.java
(original)
+++
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/SaveCrawlingDroid.java
Mon Sep 10 13:23:35 2012
@@ -1,21 +1,21 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.droids.examples;
import java.util.Queue;
@@ -27,20 +27,23 @@ import org.apache.droids.robot.crawler.C
import org.apache.droids.robot.crawler.CrawlingWorker;
public class SaveCrawlingDroid extends CrawlingDroid {
- private final Handler defaultHandler;
+ private final Handler defaultHandler;
- public SaveCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster,
final Handler defaultHandlerForWorkerCreation ) {
- super(queue, taskMaster);
-
- assert( defaultHandlerForWorkerCreation != null );
- this.defaultHandler = defaultHandlerForWorkerCreation;
- }
-
- @Override
- public Worker<Link> getNewWorker() {
- final CrawlingWorker worker = new CrawlingWorker( this );
-
worker.setHandlerFactory(DroidsFactory.createDefaultHandlerFactory(this.defaultHandler));
- return worker;
- }
+ public SaveCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster,
+ final Handler defaultHandlerForWorkerCreation) {
+
+ super(queue, taskMaster);
+
+ assert (defaultHandlerForWorkerCreation != null);
+ this.defaultHandler = defaultHandlerForWorkerCreation;
+ }
+
+ @Override
+ public Worker<Link> getNewWorker() {
+ final CrawlingWorker worker = new CrawlingWorker(this);
+ worker.setHandlerFactory(DroidsFactory
+
.createDefaultHandlerFactory(this.defaultHandler));
+ return worker;
+ }
}
Modified:
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/SysoutCrawlingDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/SysoutCrawlingDroid.java?rev=1382824&r1=1382823&r2=1382824&view=diff
==============================================================================
---
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/SysoutCrawlingDroid.java
(original)
+++
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/SysoutCrawlingDroid.java
Mon Sep 10 13:23:35 2012
@@ -1,21 +1,21 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.droids.examples;
import java.util.Queue;
@@ -28,15 +28,16 @@ import org.apache.droids.robot.crawler.C
public class SysoutCrawlingDroid extends CrawlingDroid {
- public SysoutCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
- super(queue, taskMaster);
- }
+ public SysoutCrawlingDroid(Queue<Link> queue, TaskMaster<Link>
taskMaster) {
+ super(queue, taskMaster);
+ }
- @Override
- public Worker<Link> getNewWorker() {
- final CrawlingWorker worker = new CrawlingWorker( this );
- worker.setHandlerFactory(DroidsFactory.createDefaultHandlerFactory(new
SysoutHandler()));
- return worker;
- }
+ @Override
+ public Worker<Link> getNewWorker() {
+ final CrawlingWorker worker = new CrawlingWorker(this);
+ worker.setHandlerFactory(DroidsFactory
+ .createDefaultHandlerFactory(new
SysoutHandler()));
+ return worker;
+ }
}
Modified:
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java?rev=1382824&r1=1382823&r2=1382824&view=diff
==============================================================================
---
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java
(original)
+++
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java
Mon Sep 10 13:23:35 2012
@@ -48,95 +48,96 @@ import org.apache.http.params.HttpProtoc
import org.apache.http.protocol.HTTP;
/**
- * Simple Droids runtime that wires various components together in Java code
+ * Simple Droids runtime that wires various components together in Java code
* without using a DI framework.
*
*/
public class SimpleRuntime {
-
- private SimpleRuntime(){
- }
-
- public static void main(String[] args) throws Exception {
-
- if (args.length < 1) {
- System.out.println("Please specify a URL to crawl");
- System.exit(-1);
- }
- String targetURL = args[0];
-
- // Create parser factory. Support basic HTML markup only
- ParserFactory parserFactory = new ParserFactory();
- TikaDocumentParser tikaParser = new TikaDocumentParser();
- parserFactory.getMap().put("text/html", tikaParser);
-
- // Create protocol factory. Support HTTP/S only.
- ProtocolFactory protocolFactory = new ProtocolFactory();
-
- // Create and configure HTTP client
- HttpParams params = new BasicHttpParams();
- HttpProtocolParamBean hppb = new HttpProtocolParamBean(params);
- HttpConnectionParamBean hcpb = new HttpConnectionParamBean(params);
- ConnManagerParamBean cmpb = new ConnManagerParamBean(params);
-
- // Set protocol parametes
- hppb.setVersion(HttpVersion.HTTP_1_1);
- hppb.setContentCharset(HTTP.ISO_8859_1);
- hppb.setUseExpectContinue(true);
- // Set connection parameters
- hcpb.setStaleCheckingEnabled(false);
- // Set connection manager parameters
- ConnPerRouteBean connPerRouteBean = new ConnPerRouteBean();
- connPerRouteBean.setDefaultMaxPerRoute(2);
- cmpb.setConnectionsPerRoute(connPerRouteBean);
-
- DroidsHttpClient httpclient = new DroidsHttpClient(params);
-
- HttpProtocol httpProtocol = new HttpProtocol(httpclient);
- protocolFactory.getMap().put("http", httpProtocol);
- protocolFactory.getMap().put("https", httpProtocol);
-
- // Create URL filter factory.
- URLFiltersFactory filtersFactory = new URLFiltersFactory();
- RegexURLFilter defaultURLFilter = new RegexURLFilter();
- defaultURLFilter.setFile("classpath:/regex-urlfilter.txt");
- filtersFactory.getMap().put("default", defaultURLFilter);
-
- // Create handler factory. Provide sysout handler only.
- HandlerFactory handlerFactory = new HandlerFactory();
- SysoutHandler defaultHandler = new SysoutHandler();
- handlerFactory.getMap().put("default", defaultHandler);
-
- // Create droid factory. Leave it empty for now.
- DroidFactory<Link> droidFactory = new DroidFactory<Link>();
-
- // Create default droid
- SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
- simpleDelayTimer.setDelayMillis(100);
-
- Queue<Link> simpleQueue = new LinkedList<Link>();
-
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
- taskMaster.setDelayTimer( simpleDelayTimer );
- taskMaster.setExceptionHandler( new DefaultTaskExceptionHandler() );
-
- CrawlingDroid helloCrawler = new SysoutCrawlingDroid( simpleQueue,
taskMaster );
- helloCrawler.setFiltersFactory(filtersFactory);
- helloCrawler.setParserFactory(parserFactory);
- helloCrawler.setProtocolFactory(protocolFactory);
-
- Collection<String> initialLocations = new ArrayList<String>();
- initialLocations.add( targetURL );
- helloCrawler.setInitialLocations(initialLocations);
-
- // Initialize and start the crawler
- helloCrawler.init();
- helloCrawler.start();
-
- // Await termination
- helloCrawler.getTaskMaster().awaitTermination(0, TimeUnit.MILLISECONDS);
- // Shut down the HTTP connection manager
- httpclient.getConnectionManager().shutdown();
- }
+
+ private SimpleRuntime() {
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ if (args.length < 1) {
+ System.out.println("Please specify a URL to crawl");
+ System.exit(-1);
+ }
+ String targetURL = args[0];
+
+ // Create parser factory. Support basic HTML markup only
+ ParserFactory parserFactory = new ParserFactory();
+ TikaDocumentParser tikaParser = new TikaDocumentParser();
+ parserFactory.getMap().put("text/html", tikaParser);
+
+ // Create protocol factory. Support HTTP/S only.
+ ProtocolFactory protocolFactory = new ProtocolFactory();
+
+ // Create and configure HTTP client
+ HttpParams params = new BasicHttpParams();
+ HttpProtocolParamBean hppb = new HttpProtocolParamBean(params);
+ HttpConnectionParamBean hcpb = new
HttpConnectionParamBean(params);
+ ConnManagerParamBean cmpb = new ConnManagerParamBean(params);
+
+ // Set protocol parametes
+ hppb.setVersion(HttpVersion.HTTP_1_1);
+ hppb.setContentCharset(HTTP.ISO_8859_1);
+ hppb.setUseExpectContinue(true);
+ // Set connection parameters
+ hcpb.setStaleCheckingEnabled(false);
+ // Set connection manager parameters
+ ConnPerRouteBean connPerRouteBean = new ConnPerRouteBean();
+ connPerRouteBean.setDefaultMaxPerRoute(2);
+ cmpb.setConnectionsPerRoute(connPerRouteBean);
+
+ DroidsHttpClient httpclient = new DroidsHttpClient(params);
+
+ HttpProtocol httpProtocol = new HttpProtocol(httpclient);
+ protocolFactory.getMap().put("http", httpProtocol);
+ protocolFactory.getMap().put("https", httpProtocol);
+
+ // Create URL filter factory.
+ URLFiltersFactory filtersFactory = new URLFiltersFactory();
+ RegexURLFilter defaultURLFilter = new RegexURLFilter();
+ defaultURLFilter.setFile("classpath:/regex-urlfilter.txt");
+ filtersFactory.getMap().put("default", defaultURLFilter);
+
+ // Create handler factory. Provide sysout handler only.
+ HandlerFactory handlerFactory = new HandlerFactory();
+ SysoutHandler defaultHandler = new SysoutHandler();
+ handlerFactory.getMap().put("default", defaultHandler);
+
+ // Create droid factory. Leave it empty for now.
+ DroidFactory<Link> droidFactory = new DroidFactory<Link>();
+
+ // Create default droid
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+ Queue<Link> simpleQueue = new LinkedList<Link>();
+
+ SequentialTaskMaster<Link> taskMaster = new
SequentialTaskMaster<Link>();
+ taskMaster.setDelayTimer(simpleDelayTimer);
+ taskMaster.setExceptionHandler(new
DefaultTaskExceptionHandler());
+
+ CrawlingDroid helloCrawler = new
SysoutCrawlingDroid(simpleQueue,
+ taskMaster);
+ helloCrawler.setFiltersFactory(filtersFactory);
+ helloCrawler.setParserFactory(parserFactory);
+ helloCrawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add(targetURL);
+ helloCrawler.setInitialLocations(initialLocations);
+
+ // Initialize and start the crawler
+ helloCrawler.init();
+ helloCrawler.start();
+
+ // Await termination
+ helloCrawler.getTaskMaster().awaitTermination(0,
TimeUnit.MILLISECONDS);
+ // Shut down the HTTP connection manager
+ httpclient.getConnectionManager().shutdown();
+ }
}
Modified:
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/handler/ExceptionReportHandler.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/handler/ExceptionReportHandler.java?rev=1382824&r1=1382823&r2=1382824&view=diff
==============================================================================
---
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/handler/ExceptionReportHandler.java
(original)
+++
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/handler/ExceptionReportHandler.java
Mon Sep 10 13:23:35 2012
@@ -27,13 +27,14 @@ import org.apache.droids.handle.ReportHa
public class ExceptionReportHandler extends ReportHandler {
- public ExceptionReportHandler() {
- super();
- }
+ public ExceptionReportHandler() {
+ super();
+ }
- @Override
- public void handle(URI uri, ContentEntity entity) throws IOException,
DroidsException {
- super.handle(uri, entity);
- }
+ @Override
+ public void handle(URI uri, ContentEntity entity) throws IOException,
+ DroidsException {
+ super.handle(uri, entity);
+ }
}
Modified:
incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java?rev=1382824&r1=1382823&r2=1382824&view=diff
==============================================================================
---
incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java
(original)
+++
incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java
Mon Sep 10 13:23:35 2012
@@ -42,64 +42,68 @@ import org.mockito.Mockito;
import org.mockito.internal.stubbing.defaultanswers.ReturnsMocks;
public class TestCrawlingWorker {
- CrawlingWorker instance;
- private TikaDocumentParser htmlParser;
-
- @Before
- public void initialize() {
- final Queue<Link> queue = new LinkedList<Link>();
- final CrawlingDroid droid = createDroid(queue);
- instance = (CrawlingWorker) droid.getNewWorker();
- }
-
- private final CrawlingDroid createDroid(final Queue<Link> queue) {
- final CrawlingDroid droid = new SysoutCrawlingDroid(queue, null);
-
- final ProtocolFactory protocolFactory =
DroidsFactory.createDefaultProtocolFactory();
- droid.setProtocolFactory(protocolFactory);
-
- final ParserFactory parserFactory = parserSetup();
- droid.setParserFactory(parserFactory);
- return droid;
- }
-
- private final ParserFactory parserSetup() {
- final ParserFactory parserFactory = new ParserFactory();
-
- htmlParser = Mockito.mock(TikaDocumentParser.class, new ReturnsMocks());
-
- parserFactory.getMap().put("text/html", htmlParser);
- return parserFactory;
- }
-
- //
- @After
- public void cleanup() {
- instance = null;
- htmlParser = null;
- }
-
- //
- @Test
- public void nothingHappens() {
- // Arrange
-
- // Act
-
- // Assert
- }
-
- // execute
- @Test
- public void execute_linkIsParsed() throws DroidsException, IOException,
URISyntaxException {
- // Arrange
- final Link link = new LinkTask(null, new URI("http://www.google.com";), 1);
-
- // Act
- this.instance.execute(link);
-
- // Assert
- Mockito.verify(htmlParser).parse(Matchers.any(ContentEntity.class),
Matchers.any(Link.class));
- }
-
+ CrawlingWorker instance;
+ private TikaDocumentParser htmlParser;
+
+ @Before
+ public void initialize() {
+ final Queue<Link> queue = new LinkedList<Link>();
+ final CrawlingDroid droid = createDroid(queue);
+ instance = (CrawlingWorker) droid.getNewWorker();
+ }
+
+ private final CrawlingDroid createDroid(final Queue<Link> queue) {
+ final CrawlingDroid droid = new SysoutCrawlingDroid(queue,
null);
+
+ final ProtocolFactory protocolFactory = DroidsFactory
+ .createDefaultProtocolFactory();
+ droid.setProtocolFactory(protocolFactory);
+
+ final ParserFactory parserFactory = parserSetup();
+ droid.setParserFactory(parserFactory);
+ return droid;
+ }
+
+ private final ParserFactory parserSetup() {
+ final ParserFactory parserFactory = new ParserFactory();
+
+ htmlParser = Mockito.mock(TikaDocumentParser.class, new
ReturnsMocks());
+
+ parserFactory.getMap().put("text/html", htmlParser);
+ return parserFactory;
+ }
+
+ //
+ @After
+ public void cleanup() {
+ instance = null;
+ htmlParser = null;
+ }
+
+ //
+ @Test
+ public void nothingHappens() {
+ // Arrange
+
+ // Act
+
+ // Assert
+ }
+
+ // execute
+ @Test
+ public void execute_linkIsParsed() throws DroidsException, IOException,
+ URISyntaxException {
+ // Arrange
+ final Link link = new LinkTask(null, new
URI("http://www.google.com";),
+ 1);
+
+ // Act
+ this.instance.execute(link);
+
+ // Assert
+
Mockito.verify(htmlParser).parse(Matchers.any(ContentEntity.class),
+ Matchers.any(Link.class));
+ }
+
}
Modified:
incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestSimpleDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestSimpleDroid.java?rev=1382824&r1=1382823&r2=1382824&view=diff
==============================================================================
---
incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestSimpleDroid.java
(original)
+++
incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestSimpleDroid.java
Mon Sep 10 13:23:35 2012
@@ -36,81 +36,81 @@ import org.junit.Test;
public class TestSimpleDroid {
- protected LocalHttpServer testserver;
+ protected LocalHttpServer testserver;
- @Before
- public void initializeLocalTestServer() {
- this.testserver = new LocalHttpServer();
- }
-
- @After
- public void shutdownLocalTestServer() throws IOException {
- this.testserver.stop();
- }
-
- @Test
- public void testBasicCrawling() throws Exception {
- this.testserver.register("*", new ResourceHandler());
- this.testserver.start();
-
- String baseURI = "http:/" + this.testserver.getServiceAddress();
- String targetURI = baseURI + "/start_html";
-
- Droid<Link> droid =
DroidsFactory.createSimpleReportCrawlingDroid(targetURI);
-
- droid.init();
- droid.start();
-
- while (!droid.getTaskMaster().awaitTermination(250L,
TimeUnit.MILLISECONDS))
- ;
-
- Assert.assertFalse(ReportHandler.getReport().isEmpty());
- Assert.assertEquals(5, ReportHandler.getReport().size());
- Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/start_html"));
- Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page1_html"));
- Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page2_html"));
- Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page3_html"));
- Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page4_html"));
-
- ReportHandler.recycle();
- }
-
- @Test
- public void testTerminateCrawlingOnException() throws Exception {
- this.testserver.register("*", new ResourceHandler());
- this.testserver.start();
-
- String baseURI = "http:/" + this.testserver.getServiceAddress();
- String targetURI = baseURI + "/start_html";
-
- Droid<Link> droid =
DroidsFactory.createSimpleExceptionCrawlingDroid(targetURI);
-
- TaskMaster<Link> taskMaster = (TaskMaster<Link>) droid.getTaskMaster();
- taskMaster.setExceptionHandler(new TaskExceptionHandler() {
-
- public TaskExceptionResult handleException(Exception ex) {
- if (ex instanceof RuntimeException) {
- return TaskExceptionResult.FATAL;
- }
- return TaskExceptionResult.WARN;
- }
-
- });
-
- droid.init();
- droid.start();
- while (!droid.getTaskMaster().awaitTermination(250L,
TimeUnit.MILLISECONDS))
- ;
-
- Assert.assertFalse(ReportHandler.getReport().isEmpty());
- Assert.assertEquals(5, ReportHandler.getReport().size());
- Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/start_html"));
- Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page1_html"));
- Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page2_html"));
- Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page3_html"));
- Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page4_html"));
+ @Before
+ public void initializeLocalTestServer() {
+ this.testserver = new LocalHttpServer();
+ }
+
+ @After
+ public void shutdownLocalTestServer() throws IOException {
+ this.testserver.stop();
+ }
+
+ @Test
+ public void testBasicCrawling() throws Exception {
+ this.testserver.register("*", new ResourceHandler());
+ this.testserver.start();
+
+ String baseURI = "http:/" + this.testserver.getServiceAddress();
+ String targetURI = baseURI + "/start_html";
+
+ Droid<Link> droid =
DroidsFactory.createSimpleReportCrawlingDroid(targetURI);
+
+ droid.init();
+ droid.start();
+
+ while (!droid.getTaskMaster().awaitTermination(250L,
TimeUnit.MILLISECONDS))
+ ;
+
+ Assert.assertFalse(ReportHandler.getReport().isEmpty());
+ Assert.assertEquals(5, ReportHandler.getReport().size());
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/start_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page1_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page2_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page3_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page4_html"));
+
+ ReportHandler.recycle();
+ }
+
+ @Test
+ public void testTerminateCrawlingOnException() throws Exception {
+ this.testserver.register("*", new ResourceHandler());
+ this.testserver.start();
+
+ String baseURI = "http:/" + this.testserver.getServiceAddress();
+ String targetURI = baseURI + "/start_html";
+
+ Droid<Link> droid =
DroidsFactory.createSimpleExceptionCrawlingDroid(targetURI);
+
+ TaskMaster<Link> taskMaster = (TaskMaster<Link>)
droid.getTaskMaster();
+ taskMaster.setExceptionHandler(new TaskExceptionHandler() {
+
+ public TaskExceptionResult handleException(Exception
ex) {
+ if (ex instanceof RuntimeException) {
+ return TaskExceptionResult.FATAL;
+ }
+ return TaskExceptionResult.WARN;
+ }
+
+ });
+
+ droid.init();
+ droid.start();
+ while (!droid.getTaskMaster().awaitTermination(250L,
TimeUnit.MILLISECONDS))
+ ;
+
+ Assert.assertFalse(ReportHandler.getReport().isEmpty());
+ Assert.assertEquals(5, ReportHandler.getReport().size());
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/start_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page1_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page2_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page3_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page4_html"));
- ReportHandler.recycle();
- }
+ ReportHandler.recycle();
+ }
}
