Author: lewismc
Date: Mon Sep 22 14:18:53 2014
New Revision: 1626786
URL: http://svn.apache.org/r1626786
Log:
NUTCH-841 Create a Wicket-based Web Application for Nutch
Added:
nutch/branches/2.x/src/java/org/apache/nutch/api/model/request/SeedList.java
nutch/branches/2.x/src/java/org/apache/nutch/api/model/request/SeedUrl.java
nutch/branches/2.x/src/java/org/apache/nutch/api/resources/SeedResource.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/
nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiApplication.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiApplication.properties
nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiServer.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/NutchClient.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/NutchClientFactory.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/CrawlingCycle.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/CrawlingCycleListener.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/NutchClientImpl.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommand.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandBuilder.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandExecutor.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandsBatchFactory.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/ConnectionStatus.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/Crawl.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/JobConfig.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/JobInfo.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/NutchStatus.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/CustomDaoFactory.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/CustomTableCreator.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/NutchGuiConfiguration.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/SpringConfiguration.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/model/
nutch/branches/2.x/src/java/org/apache/nutch/webui/model/NutchConfig.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/model/NutchInstance.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/model/SeedList.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/model/SeedUrl.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/AbstractBasePage.html
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/AbstractBasePage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/DashboardPage.html
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/DashboardPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/LogOutPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/SchedulingPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/SearchPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/StatisticsPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/UrlsUploadPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/UserSettingsPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/assets/
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/assets/NutchUiCssReference.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/assets/nutch-style.css
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/components/
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/components/ColorEnumLabel.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/components/ColorEnumLabelBuilder.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/components/CpmIteratorAdapter.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/crawls/
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/crawls/CrawlPanel.html
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/crawls/CrawlPanel.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/crawls/CrawlsPage.html
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/crawls/CrawlsPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/instances/
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/instances/InstancePanel.html
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/instances/InstancePanel.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/instances/InstancesPage.html
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/instances/InstancesPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/menu/
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/menu/VerticalMenu.html
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/menu/VerticalMenu.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/seed/
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/seed/SeedListsPage.html
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/seed/SeedListsPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/seed/SeedPage.html
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/seed/SeedPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/settings/
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/settings/SettingsPage.html
nutch/branches/2.x/src/java/org/apache/nutch/webui/pages/settings/SettingsPage.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/service/
nutch/branches/2.x/src/java/org/apache/nutch/webui/service/CrawlService.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/service/NutchInstanceService.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/service/NutchService.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/service/SeedListService.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/service/impl/
nutch/branches/2.x/src/java/org/apache/nutch/webui/service/impl/CrawlServiceImpl.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/service/impl/NutchInstanceServiceImpl.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/service/impl/NutchServiceImpl.java
nutch/branches/2.x/src/java/org/apache/nutch/webui/service/impl/SeedListServiceImpl.java
nutch/branches/2.x/src/test/org/apache/nutch/webui/
nutch/branches/2.x/src/test/org/apache/nutch/webui/client/
nutch/branches/2.x/src/test/org/apache/nutch/webui/client/TestCrawlCycle.java
nutch/branches/2.x/src/test/org/apache/nutch/webui/client/TestNutchClientFactory.java
nutch/branches/2.x/src/test/org/apache/nutch/webui/client/TestRemoteCommandExecutor.java
nutch/branches/2.x/src/test/org/apache/nutch/webui/client/TestRemoteCommandsBatchFactory.java
nutch/branches/2.x/src/test/org/apache/nutch/webui/service/
nutch/branches/2.x/src/test/org/apache/nutch/webui/service/NutchServiceTest.java
nutch/branches/2.x/src/test/org/apache/nutch/webui/view/
nutch/branches/2.x/src/test/org/apache/nutch/webui/view/AbstractWicketTest.java
nutch/branches/2.x/src/test/org/apache/nutch/webui/view/SpringConfigForTests.java
nutch/branches/2.x/src/test/org/apache/nutch/webui/view/TestColorEnumLabel.java
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/build.xml
nutch/branches/2.x/default.properties
nutch/branches/2.x/ivy/ivy.xml
nutch/branches/2.x/src/bin/nutch
nutch/branches/2.x/src/java/org/apache/nutch/api/NutchServer.java
nutch/branches/2.x/src/java/org/apache/nutch/api/impl/NutchServerPoolExecutor.java
nutch/branches/2.x/src/java/org/apache/nutch/api/impl/RAMJobManager.java
Modified: nutch/branches/2.x/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1626786&r1=1626785&r2=1626786&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Mon Sep 22 14:18:53 2014
@@ -2,6 +2,8 @@ Nutch Change Log
Current Development
+* NUTCH-841 Create a Wicket-based Web Application for Nutch (Fjodor Vershinin
via lewismc)
+
* NUTCH-1832 Make Nutch work without an indexer (mattmann via lewismc)
* NUTCH-1840 the describe function in SolrIndexWriter is not correct (kaveh
minooie via jnioche)
Modified: nutch/branches/2.x/build.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/build.xml?rev=1626786&r1=1626785&r2=1626786&view=diff
==============================================================================
--- nutch/branches/2.x/build.xml (original)
+++ nutch/branches/2.x/build.xml Mon Sep 22 14:18:53 2014
@@ -102,6 +102,11 @@
<compilerarg value="-Xlint:-path"/>
<classpath refid="classpath" />
</javac>
+ <copy todir="${build.classes}">
+ <fileset dir="${src.dir}" includes="**/*.html" />
+ <fileset dir="${src.dir}" includes="**/*.css" />
+ <fileset dir="${src.dir}" includes="**/*.properties" />
+ </copy>
</target>
<target name="compile-plugins" depends="init, resolve-default"
description="--> compile plugins only">
Modified: nutch/branches/2.x/default.properties
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/default.properties?rev=1626786&r1=1626785&r2=1626786&view=diff
==============================================================================
--- nutch/branches/2.x/default.properties (original)
+++ nutch/branches/2.x/default.properties Mon Sep 22 14:18:53 2014
@@ -61,7 +61,7 @@ runtime.dir=./runtime
runtime.deploy=${runtime.dir}/deploy
runtime.local=${runtime.dir}/local
-ivy.version=2.2.0
+ivy.version=2.3.0
ivy.dir=${basedir}/ivy
ivy.file=${ivy.dir}/ivy.xml
ivy.jar=${ivy.dir}/ivy-${ivy.version}.jar
Modified: nutch/branches/2.x/ivy/ivy.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/ivy/ivy.xml?rev=1626786&r1=1626785&r2=1626786&view=diff
==============================================================================
--- nutch/branches/2.x/ivy/ivy.xml (original)
+++ nutch/branches/2.x/ivy/ivy.xml Mon Sep 22 14:18:53 2014
@@ -69,7 +69,12 @@
<dependency org="com.google.guava" name="guava" rev="11.0.2" />
<dependency org="com.google.code.crawler-commons" name="crawler-commons"
rev="0.4" />
-
+
+ <dependency org="org.restlet.jse" name="org.restlet" rev="2.2.1"
conf="*->default" />
+ <dependency org="org.restlet.jse" name="org.restlet.ext.jackson"
rev="2.2.1"
+ conf="*->default" />
+ <dependency org="org.restlet.jse" name="org.restlet.ext.jaxrs" rev="2.2.1"
+ conf="*->default" />
<!--Configuration: test -->
<!--artifacts needed for testing -->
@@ -88,12 +93,9 @@
<dependency org="org.hsqldb" name="hsqldb" rev="2.2.8" conf="*->default" />
<dependency org="org.jdom" name="jdom" rev="1.1" conf="test->default"/>
-
- <dependency org="org.restlet.jse" name="org.restlet" rev="2.2.1"
conf="*->default" />
- <dependency org="org.restlet.jse" name="org.restlet.ext.jackson"
rev="2.2.1"
- conf="*->default" />
- <dependency org="org.restlet.jse" name="org.restlet.ext.jaxrs" rev="2.2.1"
- conf="*->default" />
+ <dependency org="org.mockito" name="mockito-all" rev="1.9.5"
conf="test->default"/>
+ <dependency org="org.springframework" name="spring-test"
rev="4.0.4.RELEASE" conf="test->default"/>
+
<!--================-->
<!-- Gora artifacts -->
@@ -122,10 +124,27 @@
<dependency org="org.apache.gora" name="gora-accumulo" rev="0.4"
conf="*->default" />
-->
<!-- Uncomment this to use Cassandra as Gora backend. -->
- <!--
+
<dependency org="org.apache.gora" name="gora-cassandra" rev="0.4"
conf="*->default" />
- -->
+
+ <!-- web app dependencies -->
+
+ <dependency org="org.apache.commons" name="commons-collections4"
rev="4.0" conf="*->default" />
+ <dependency org="org.springframework" name="spring-core"
rev="4.0.4.RELEASE" conf="*->default" />
+ <dependency org="org.springframework" name="spring-context"
rev="4.0.4.RELEASE" conf="*->default" />
+ <dependency org="org.springframework" name="spring-web"
rev="4.0.4.RELEASE" conf="*->default" />
+
+ <dependency org="com.sun.jersey" name="jersey-client" rev="1.8"
conf="*->default" />
+
+ <dependency org="com.j256.ormlite" name="ormlite-jdbc" rev="4.48"
conf="*->default" />
+ <dependency org="com.h2database" name="h2" rev="1.4.180"
conf="*->default" />
+ <dependency org="org.eclipse.persistence" name="javax.persistence"
rev="2.0.0" conf="*->default" />
+
+ <dependency org="org.apache.wicket" name="wicket-core" rev="6.16.0"
conf="*->default" />
+ <dependency org="org.apache.wicket" name="wicket-spring" rev="6.16.0"
conf="*->default" />
+ <dependency org="de.agilecoders.wicket" name="wicket-bootstrap-core"
rev="0.9.2" conf="*->default" />
+ <dependency org="de.agilecoders.wicket"
name="wicket-bootstrap-extensions" rev="0.9.2" conf="*->default" />
<!--global exclusion -->
<exclude module="ant" />
Modified: nutch/branches/2.x/src/bin/nutch
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/bin/nutch?rev=1626786&r1=1626785&r2=1626786&view=diff
==============================================================================
--- nutch/branches/2.x/src/bin/nutch (original)
+++ nutch/branches/2.x/src/bin/nutch Mon Sep 22 14:18:53 2014
@@ -74,6 +74,7 @@ if [ $# = 0 ]; then
echo " indexchecker check the indexing filters for a given url"
echo " plugin load a plugin and run one of its classes main()"
echo " nutchserver run a (local) Nutch server on a user defined port"
+ echo " webapp run a local Nutch web application"
echo " junit runs the given JUnit test"
echo " or"
echo " CLASSNAME run the class named CLASSNAME"
@@ -234,6 +235,8 @@ elif [ "$COMMAND" = "indexchecker" ] ; t
CLASS=org.apache.nutch.indexer.IndexingFiltersChecker
elif [ "$COMMAND" = "plugin" ] ; then
CLASS=org.apache.nutch.plugin.PluginRepository
+elif [ "$COMMAND" = "webapp" ] ; then
+CLASS=org.apache.nutch.webui.NutchUiServer
elif [ "$COMMAND" = "nutchserver" ] ; then
CLASS=org.apache.nutch.api.NutchServer
elif [ "$COMMAND" = "junit" ] ; then
Modified: nutch/branches/2.x/src/java/org/apache/nutch/api/NutchServer.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/api/NutchServer.java?rev=1626786&r1=1626785&r2=1626786&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/api/NutchServer.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/api/NutchServer.java Mon Sep
22 14:18:53 2014
@@ -42,6 +42,7 @@ import org.apache.nutch.api.resources.Ad
import org.apache.nutch.api.resources.ConfigResource;
import org.apache.nutch.api.resources.DbResource;
import org.apache.nutch.api.resources.JobResource;
+import org.apache.nutch.api.resources.SeedResource;
import org.restlet.Component;
import org.restlet.Context;
import org.restlet.data.Protocol;
@@ -118,6 +119,7 @@ public class NutchServer extends Applica
resources.add(AdminResource.class);
resources.add(ConfigResource.class);
resources.add(DbResource.class);
+ resources.add(SeedResource.class);
return resources;
}
Modified:
nutch/branches/2.x/src/java/org/apache/nutch/api/impl/NutchServerPoolExecutor.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/api/impl/NutchServerPoolExecutor.java?rev=1626786&r1=1626785&r2=1626786&view=diff
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/api/impl/NutchServerPoolExecutor.java
(original)
+++
nutch/branches/2.x/src/java/org/apache/nutch/api/impl/NutchServerPoolExecutor.java
Mon Sep 22 14:18:53 2014
@@ -100,4 +100,13 @@ public class NutchServerPoolExecutor ext
}
return jobsInfo;
}
+
+ public JobInfo getInfo(String jobId) {
+ for (JobInfo jobInfo : getAllJobs()) {
+ if(StringUtils.equals(jobId, jobInfo.getId())){
+ return jobInfo;
+ }
+ }
+ return null;
+ }
}
Modified:
nutch/branches/2.x/src/java/org/apache/nutch/api/impl/RAMJobManager.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/api/impl/RAMJobManager.java?rev=1626786&r1=1626785&r2=1626786&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/api/impl/RAMJobManager.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/api/impl/RAMJobManager.java
Mon Sep 22 14:18:53 2014
@@ -52,7 +52,7 @@ public class RAMJobManager implements Jo
@Override
public JobInfo get(String crawlId, String jobId) {
- return executor.findWorker(jobId).getInfo();
+ return executor.getInfo(jobId);
}
@Override
Added:
nutch/branches/2.x/src/java/org/apache/nutch/api/model/request/SeedList.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/api/model/request/SeedList.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/api/model/request/SeedList.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/api/model/request/SeedList.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,79 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
******************************************************************************/
+package org.apache.nutch.api.model.request;
+
+import java.io.Serializable;
+import java.util.Collection;
+
+public class SeedList implements Serializable {
+
+ private Long id;
+
+ private String name;
+
+ private Collection<SeedUrl> seedUrls;
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public Collection<SeedUrl> getSeedUrls() {
+ return seedUrls;
+ }
+
+ public void setSeedUrls(Collection<SeedUrl> seedUrls) {
+ this.seedUrls = seedUrls;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((id == null) ? 0 : id.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ SeedList other = (SeedList) obj;
+ if (id == null) {
+ if (other.id != null)
+ return false;
+ } else if (!id.equals(other.id))
+ return false;
+ return true;
+ }
+
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/api/model/request/SeedUrl.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/api/model/request/SeedUrl.java?rev=1626786&view=auto
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/api/model/request/SeedUrl.java
(added)
+++ nutch/branches/2.x/src/java/org/apache/nutch/api/model/request/SeedUrl.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,61 @@
+package org.apache.nutch.api.model.request;
+
+import java.io.Serializable;
+
+public class SeedUrl implements Serializable {
+
+ private Long id;
+
+ private SeedList seedList;
+
+ private String url;
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public SeedList getSeedList() {
+ return seedList;
+ }
+
+ public void setSeedList(SeedList seedList) {
+ this.seedList = seedList;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((id == null) ? 0 : id.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ SeedUrl other = (SeedUrl) obj;
+ if (id == null) {
+ if (other.id != null)
+ return false;
+ } else if (!id.equals(other.id))
+ return false;
+ return true;
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/api/resources/SeedResource.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/api/resources/SeedResource.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/api/resources/SeedResource.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/api/resources/SeedResource.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,108 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
******************************************************************************/
+package org.apache.nutch.api.resources;
+
+import static javax.ws.rs.core.Response.status;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Collection;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.nutch.api.model.request.SeedList;
+import org.apache.nutch.api.model.request.SeedUrl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.io.Files;
+
+@Path("/seed")
+public class SeedResource extends AbstractResource {
+ private static final Logger log =
LoggerFactory.getLogger(AdminResource.class);
+
+ @POST
+ @Path("/create")
+ @Consumes(MediaType.APPLICATION_JSON)
+ /**
+ * Method creates seed list file and returns temorary directory path
+ * @param seedList
+ * @return
+ */
+ public String createSeedFile(SeedList seedList) {
+ if (seedList == null) {
+ throw new WebApplicationException(Response.status(Status.BAD_REQUEST)
+ .entity("Seed list cannot be empty!").build());
+ }
+ File seedFile = createSeedFile();
+ BufferedWriter writer = getWriter(seedFile);
+
+ Collection<SeedUrl> seedUrls = seedList.getSeedUrls();
+ if (CollectionUtils.isNotEmpty(seedUrls)) {
+ for (SeedUrl seedUrl : seedUrls) {
+ writeUrl(writer, seedUrl);
+ }
+ }
+
+ return seedFile.getParent();
+ }
+
+ private void writeUrl(BufferedWriter writer, SeedUrl seedUrl) {
+ try {
+ writer.write(seedUrl.getUrl());
+ writer.newLine();
+ writer.flush();
+ } catch (IOException e) {
+ throw handleException(e);
+ }
+ }
+
+ private BufferedWriter getWriter(File seedFile) {
+ try {
+ return new BufferedWriter(new FileWriter(seedFile));
+ } catch (FileNotFoundException e) {
+ throw handleException(e);
+ } catch (IOException e) {
+ throw handleException(e);
+ }
+ }
+
+ private File createSeedFile() {
+ try {
+ return File.createTempFile("seed", ".txt", Files.createTempDir());
+ } catch (IOException e) {
+ throw handleException(e);
+ }
+ }
+
+ private RuntimeException handleException(Exception e) {
+ log.error("Cannot create seed file!", e);
+ return new
WebApplicationException(status(Status.INTERNAL_SERVER_ERROR).entity(
+ "Cannot create seed file!").build());
+ }
+
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiApplication.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiApplication.java?rev=1626786&view=auto
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiApplication.java
(added)
+++ nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiApplication.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui;
+
+import org.apache.nutch.webui.pages.DashboardPage;
+import org.apache.nutch.webui.pages.assets.NutchUiCssReference;
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.protocol.http.WebApplication;
+import org.apache.wicket.spring.injection.annot.SpringComponentInjector;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Component;
+
+import de.agilecoders.wicket.core.Bootstrap;
+import
de.agilecoders.wicket.core.markup.html.themes.bootstrap.BootstrapCssReference;
+import de.agilecoders.wicket.core.settings.BootstrapSettings;
+import de.agilecoders.wicket.core.settings.SingleThemeProvider;
+import de.agilecoders.wicket.core.settings.Theme;
+import
de.agilecoders.wicket.extensions.markup.html.bootstrap.icon.FontAwesomeCssReference;
+
+@Component
+public class NutchUiApplication extends WebApplication implements
ApplicationContextAware {
+ private static final String THEME_NAME = "bootstrap";
+ private ApplicationContext context;
+
+ /**
+ * @see org.apache.wicket.Application#getHomePage()
+ */
+ @Override
+ public Class<? extends WebPage> getHomePage() {
+ return DashboardPage.class;
+ }
+
+ /**
+ * @see org.apache.wicket.Application#init()
+ */
+ @Override
+ public void init() {
+ super.init();
+ BootstrapSettings settings = new BootstrapSettings();
+ Bootstrap.install(this, settings);
+ configureTheme(settings);
+
+ getComponentInstantiationListeners().add(new SpringComponentInjector(this,
context));
+ }
+
+ private void configureTheme(BootstrapSettings settings) {
+ Theme theme = new Theme(THEME_NAME, BootstrapCssReference.instance(),
+ FontAwesomeCssReference.instance(), NutchUiCssReference.instance());
+ settings.setThemeProvider(new SingleThemeProvider(theme));
+ }
+
+ @Override
+ public void setApplicationContext(ApplicationContext applicationContext)
throws BeansException {
+ this.context = applicationContext;
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiApplication.properties
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiApplication.properties?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiApplication.properties
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiApplication.properties
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,63 @@
+#############################################################################
+#Licensed to the Apache Software Foundation (ASF) under one or more
+#contributor license agreements. See the NOTICE file distributed with
+#this work for additional information regarding copyright ownership.
+#The ASF licenses this file to You under the Apache License, Version 2.0
+#(the "License"); you may not use this file except in compliance with
+#the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#############################################################################
+
+navbar.menu.dashboard = Dashboard
+navbar.menu.statistics = Statistics
+navbar.menu.instances = Instances
+navbar.menu.settings = Settings
+navbar.menu.crawls = Crawls
+navbar.menu.scheduling = Scheduling
+navbar.menu.search = Search
+navbar.menu.url = URLs upload
+navbar.menu.seedLists = Seed lists
+
+page.header.seedList = Seed list
+
+navbar.userMenu.settings = Settings
+navbar.userMenu.logout = Log out
+
+menu.settings=Settings
+menu.instances=Instances
+
+connected=Connected
+disconnected=Disconnected
+
+##ENUMS
+ConnectionStatus.CONNECTING=Connecting
+ConnectionStatus.CONNECTED=Connected
+ConnectionStatus.DISCONNECTED=Disconnected
+
+CrawlStatus.NEW=New
+CrawlStatus.ERROR=Error
+CrawlStatus.CRAWLING=Crawling
+CrawlStatus.FINISHED=Finished
+
+instances=Instances
+instances.header.name=Instance name
+instances.header.hostname=Hostname
+instances.header.status=Status
+instances.header.username=Username
+instances.label.name=Instance name
+instances.label.hostname=Hostname
+instances.label.port=Port
+instances.label.username=Username
+instances.label.password=Password
+instances.buttons.addInstance=Add instance
+
+settings=Settings
+settings.header.name = Name
+settings.header.value = Value
\ No newline at end of file
Added: nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiServer.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiServer.java?rev=1626786&view=auto
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiServer.java
(added)
+++ nutch/branches/2.x/src/java/org/apache/nutch/webui/NutchUiServer.java Mon
Sep 22 14:18:53 2014
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui;
+
+import org.apache.wicket.protocol.http.WicketFilter;
+import org.apache.wicket.spring.SpringWebApplicationFactory;
+import org.mortbay.jetty.Handler;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.servlet.Context;
+import org.mortbay.jetty.servlet.DefaultServlet;
+import org.mortbay.jetty.servlet.FilterHolder;
+import org.springframework.web.context.ContextLoaderListener;
+import org.springframework.web.context.WebApplicationContext;
+import org.springframework.web.context.request.RequestContextListener;
+import
org.springframework.web.context.support.AnnotationConfigWebApplicationContext;
+
+public class NutchUiServer {
+ private static final String APP_FACTORY_NAME =
SpringWebApplicationFactory.class.getName();
+ private static final String CONFIG_LOCATION = "org.apache.nutch.webui";
+
+ public static void main(String[] args) throws Exception {
+ startServer();
+ }
+
+ private static void startServer() throws Exception, InterruptedException {
+ Server server = new Server(8080);
+ Context context = new Context(server, "/", Context.SESSIONS);
+ context.addServlet(DefaultServlet.class, "/*");
+
+ context.addEventListener(new ContextLoaderListener(getContext()));
+ context.addEventListener(new RequestContextListener());
+
+ WicketFilter filter = new WicketFilter();
+ filter.setFilterPath("/");
+ FilterHolder holder = new FilterHolder(filter);
+ holder.setInitParameter("applicationFactoryClassName", APP_FACTORY_NAME);
+ context.addFilter(holder, "/*", Handler.DEFAULT);
+
+ server.setHandler(context);
+ server.start();
+ server.join();
+ }
+
+ private static WebApplicationContext getContext() {
+ AnnotationConfigWebApplicationContext context = new
AnnotationConfigWebApplicationContext();
+ context.setConfigLocation(CONFIG_LOCATION);
+ return context;
+ }
+
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/NutchClient.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/NutchClient.java?rev=1626786&view=auto
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/webui/client/NutchClient.java
(added)
+++ nutch/branches/2.x/src/java/org/apache/nutch/webui/client/NutchClient.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client;
+
+import java.util.Map;
+
+import org.apache.nutch.webui.client.model.ConnectionStatus;
+import org.apache.nutch.webui.client.model.JobConfig;
+import org.apache.nutch.webui.client.model.JobInfo;
+import org.apache.nutch.webui.client.model.NutchStatus;
+import org.apache.nutch.webui.model.NutchInstance;
+import org.apache.nutch.webui.model.SeedList;
+
+public interface NutchClient {
+
+ public NutchInstance getNutchInstance();
+
+ public NutchStatus getNutchStatus();
+
+ public ConnectionStatus getConnectionStatus();
+
+ public String executeJob(JobConfig jobConfig);
+
+ public JobInfo getJobInfo(String jobId);
+
+ public Map<String, String> getNutchConfig(String config);
+
+ /**
+ * Create seed list and return seed directory location
+ *
+ * @param seedList
+ * @return
+ */
+ public String createSeed(SeedList seedList);
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/NutchClientFactory.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/NutchClientFactory.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/NutchClientFactory.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/NutchClientFactory.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client;
+
+import java.util.concurrent.ExecutionException;
+
+import org.apache.nutch.webui.client.impl.NutchClientImpl;
+import org.apache.nutch.webui.model.NutchInstance;
+import org.springframework.stereotype.Component;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+@Component
+public class NutchClientFactory {
+ private LoadingCache<NutchInstance, NutchClient> cache;
+
+ public NutchClientFactory() {
+ cache = CacheBuilder.newBuilder().build(new NutchClientCacheLoader());
+ }
+
+ public NutchClient getClient(NutchInstance instance) {
+ try {
+ return cache.get(instance);
+ } catch (ExecutionException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private static class NutchClientCacheLoader extends
CacheLoader<NutchInstance, NutchClient> {
+ @Override
+ public NutchClient load(NutchInstance key) throws Exception {
+ return new NutchClientImpl(key);
+ }
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/CrawlingCycle.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/CrawlingCycle.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/CrawlingCycle.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/CrawlingCycle.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import java.util.List;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.nutch.webui.client.model.Crawl;
+import org.apache.nutch.webui.client.model.JobInfo;
+import org.apache.nutch.webui.client.model.JobInfo.State;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+/**
+ * This class implements crawl cycle as in crawl script
+ *
+ * @author feodor
+ *
+ */
+public class CrawlingCycle {
+ private Logger log = LoggerFactory.getLogger(CrawlingCycle.class);
+
+ private CrawlingCycleListener listener;
+ private RemoteCommandExecutor executor;
+ private Crawl crawl;
+
+ private List<RemoteCommand> remoteCommands;
+ private List<RemoteCommand> executedCommands = Lists.newArrayList();
+
+ public CrawlingCycle(CrawlingCycleListener listener, RemoteCommandExecutor
executor, Crawl crawl,
+ List<RemoteCommand> commands) {
+ this.listener = listener;
+ this.executor = executor;
+ this.crawl = crawl;
+ this.remoteCommands = commands;
+ }
+
+ public synchronized void executeCrawlCycle() {
+ listener.crawlingStarted(crawl);
+
+ for (RemoteCommand command : remoteCommands) {
+ JobInfo jobInfo = executor.executeRemoteJob(command);
+ command.setJobInfo(jobInfo);
+
+ log.info("Executed remote command data: {}", command);
+
+ if (jobInfo.getState() == State.FAILED) {
+ listener.onCrawlError(crawl, jobInfo.getMsg());
+ return;
+ }
+
+ executedCommands.add(command);
+ listener.commandExecuted(crawl, command, calculateProgress());
+ }
+ listener.crawlingFinished(crawl);
+ }
+
+ private int calculateProgress() {
+ if (CollectionUtils.isEmpty(remoteCommands)) {
+ return 0;
+ }
+ return (int) ((float) executedCommands.size() / (float)
remoteCommands.size() * 100);
+ }
+
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/CrawlingCycleListener.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/CrawlingCycleListener.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/CrawlingCycleListener.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/CrawlingCycleListener.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import org.apache.nutch.webui.client.model.Crawl;
+
+public interface CrawlingCycleListener {
+
+ void crawlingStarted(Crawl crawl);
+
+ void onCrawlError(Crawl crawl, String msg);
+
+ void commandExecuted(Crawl crawl, RemoteCommand command, int progress);
+
+ void crawlingFinished(Crawl crawl);
+
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/NutchClientImpl.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/NutchClientImpl.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/NutchClientImpl.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/NutchClientImpl.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+
+import java.util.Map;
+
+import org.apache.nutch.webui.client.NutchClient;
+import org.apache.nutch.webui.client.model.ConnectionStatus;
+import org.apache.nutch.webui.client.model.JobConfig;
+import org.apache.nutch.webui.client.model.JobInfo;
+import org.apache.nutch.webui.client.model.NutchStatus;
+import org.apache.nutch.webui.model.NutchInstance;
+import org.apache.nutch.webui.model.SeedList;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+
+public class NutchClientImpl implements NutchClient {
+ private Client client;
+ private WebResource nutchResource;
+ private NutchInstance instance;
+
+ public NutchClientImpl(NutchInstance instance) {
+ this.instance = instance;
+ createClient();
+ }
+
+ public void createClient() {
+ ClientConfig clientConfig = new DefaultClientConfig();
+ clientConfig.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING,
true);
+ this.client = Client.create(clientConfig);
+ this.nutchResource = client.resource(instance.getUrl());
+ }
+
+ @Override
+ public NutchStatus getNutchStatus() {
+ return
nutchResource.path("/admin").type(APPLICATION_JSON).get(NutchStatus.class);
+ }
+
+ @Override
+ public ConnectionStatus getConnectionStatus() {
+
+ getNutchStatus();
+ return ConnectionStatus.CONNECTED;
+ // TODO implement disconnected status
+ }
+
+ @Override
+ public String executeJob(JobConfig jobConfig) {
+ return
nutchResource.path("/job/create").type(APPLICATION_JSON).post(String.class,
jobConfig);
+ }
+
+ @Override
+ public JobInfo getJobInfo(String jobId) {
+ return nutchResource.path("/job/" +
jobId).type(APPLICATION_JSON).get(JobInfo.class);
+ }
+
+ @Override
+ public NutchInstance getNutchInstance() {
+ return instance;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map<String, String> getNutchConfig(String config) {
+ return nutchResource.path("/config/" +
config).type(APPLICATION_JSON).get(Map.class);
+ }
+
+ @Override
+ public String createSeed(SeedList seedList) {
+ return
nutchResource.path("/seed/create").type(APPLICATION_JSON).post(String.class,
seedList);
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommand.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommand.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommand.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommand.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import java.io.Serializable;
+import java.text.MessageFormat;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nutch.webui.client.model.JobConfig;
+import org.apache.nutch.webui.client.model.JobInfo;
+import org.joda.time.Duration;
+
+public class RemoteCommand implements Serializable {
+ private JobConfig jobConfig;
+ private JobInfo jobInfo = new JobInfo();
+ private Duration timeout;
+
+ /**
+ * Use {@link RemoteCommandBuilder} instead
+ */
+ @SuppressWarnings("unused")
+ private RemoteCommand() {
+ }
+
+ public RemoteCommand(JobConfig jobConfig) {
+ this.jobConfig = jobConfig;
+ }
+
+ public JobConfig getJobConfig() {
+ return jobConfig;
+ }
+
+ public void setJobConfig(JobConfig jobConfig) {
+ this.jobConfig = jobConfig;
+ }
+
+ public JobInfo getJobInfo() {
+ return jobInfo;
+ }
+
+ public void setJobInfo(JobInfo jobInfo) {
+ this.jobInfo = jobInfo;
+ }
+
+ public Duration getTimeout() {
+ return timeout;
+ }
+
+ public void setTimeout(Duration timeout) {
+ this.timeout = timeout;
+ }
+
+ @Override
+ public String toString() {
+ String statusInfo = StringUtils.EMPTY;
+ if (jobInfo != null) {
+ statusInfo = MessageFormat.format("{0}", jobInfo.getState());
+ }
+ return MessageFormat.format("{0} status: {1}", jobConfig.getType(),
statusInfo);
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandBuilder.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandBuilder.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandBuilder.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandBuilder.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import org.apache.nutch.webui.client.model.JobConfig;
+import org.apache.nutch.webui.client.model.JobInfo.JobType;
+import org.joda.time.Duration;
+
+public class RemoteCommandBuilder {
+ private JobConfig jobConfig = new JobConfig();
+ private Duration timeout = Duration.standardSeconds(10);
+
+ private RemoteCommandBuilder() {
+ }
+
+ public static RemoteCommandBuilder instance(JobType jobType) {
+ return new RemoteCommandBuilder().withJobType(jobType);
+ }
+
+ public RemoteCommandBuilder withJobType(JobType jobType) {
+ jobConfig.setType(jobType);
+ return this;
+ }
+
+ public RemoteCommandBuilder withConfigId(String configId) {
+ jobConfig.setConfId(configId);
+ return this;
+ }
+ public RemoteCommandBuilder withCrawlId(String crawlId) {
+ jobConfig.setCrawlId(crawlId);
+ return this;
+ }
+ public RemoteCommandBuilder withArgument(String key, String value) {
+ jobConfig.setArgument(key, value);
+ return this;
+ }
+
+ public RemoteCommandBuilder withTimeout(Duration timeout) {
+ this.timeout = timeout;
+ return this;
+ }
+
+ public RemoteCommand build() {
+ RemoteCommand remoteCommand = new RemoteCommand(jobConfig);
+ remoteCommand.setTimeout(timeout);
+ return remoteCommand;
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandExecutor.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandExecutor.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandExecutor.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandExecutor.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.nutch.webui.client.NutchClient;
+import org.apache.nutch.webui.client.model.JobInfo;
+import org.apache.nutch.webui.client.model.JobInfo.State;
+import org.joda.time.DateTimeConstants;
+import org.joda.time.Duration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class executes remote job and waits for success/failure result
+ *
+ * @author feodor
+ *
+ */
+public class RemoteCommandExecutor {
+ private Logger log = LoggerFactory.getLogger(RemoteCommandExecutor.class);
+
+ private static final int DEFAULT_TIMEOUT_SEC = 60;
+ private Duration requestDelay = new Duration(500);
+
+ private NutchClient client;
+ private ExecutorService executor;
+
+ public RemoteCommandExecutor(NutchClient client) {
+ this.client = client;
+ this.executor = Executors.newSingleThreadExecutor();
+ }
+
+ public JobInfo executeRemoteJob(RemoteCommand command) {
+ try {
+ String jobId = client.executeJob(command.getJobConfig());
+ Future<JobInfo> chekerFuture = executor.submit(new
JobStateChecker(jobId));
+ return chekerFuture.get(getTimeout(command), TimeUnit.MILLISECONDS);
+ } catch (Exception e) {
+ log.error("Remote command failed", e);
+ JobInfo jobInfo = new JobInfo();
+ jobInfo.setState(State.FAILED);
+ jobInfo.setMsg(ExceptionUtils.getStackTrace(e));
+ return jobInfo;
+ }
+ }
+
+ private long getTimeout(RemoteCommand command) {
+ if (command.getTimeout() == null) {
+ return DEFAULT_TIMEOUT_SEC * DateTimeConstants.MILLIS_PER_SECOND;
+ }
+ return command.getTimeout().getMillis();
+ }
+
+ public void setRequestDelay(Duration requestDelay) {
+ this.requestDelay = requestDelay;
+ }
+
+ public class JobStateChecker implements Callable<JobInfo> {
+
+ private String jobId;
+
+ public JobStateChecker(String jobId) {
+ this.jobId = jobId;
+ }
+
+ @Override
+ public JobInfo call() throws Exception {
+ while (!Thread.interrupted()) {
+ JobInfo jobInfo = client.getJobInfo(jobId);
+ checkState(jobInfo != null, "Cannot get job info!");
+
+ State state = jobInfo.getState();
+ checkState(state != null, "Unknown job state!");
+
+ if (state == State.RUNNING || state == State.ANY || state ==
State.IDLE) {
+ Thread.sleep(requestDelay.getMillis());
+ continue;
+ }
+
+ return jobInfo;
+ }
+ return null;
+ }
+
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandsBatchFactory.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandsBatchFactory.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandsBatchFactory.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/impl/RemoteCommandsBatchFactory.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.nutch.webui.client.model.Crawl;
+import org.apache.nutch.webui.client.model.JobInfo.JobType;
+import org.joda.time.Duration;
+import org.springframework.beans.factory.config.BeanDefinition;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import com.google.common.collect.Lists;
+
+@Component
+@Scope(BeanDefinition.SCOPE_PROTOTYPE)
+public class RemoteCommandsBatchFactory {
+
+ private List<RemoteCommand> remoteCommands;
+ private Crawl crawl;
+
+ private String batchId;
+
+ public List<RemoteCommand> createCommands(Crawl crawl) {
+ this.crawl = crawl;
+ this.remoteCommands = Lists.newArrayList();
+
+ remoteCommands.add(inject());
+ for (int i = 0; i < crawl.getNumberOfRounds(); i++) {
+ remoteCommands.addAll(createBatchCommands());
+ }
+ return remoteCommands;
+ }
+
+ private List<RemoteCommand> createBatchCommands() {
+ this.batchId = UUID.randomUUID().toString();
+ List<RemoteCommand> batchCommands = Lists.newArrayList();
+
+ batchCommands.add(createGenerateCommand());
+ batchCommands.add(createFetchCommand());
+ batchCommands.add(createParseCommand());
+ batchCommands.add(createUpdateDbCommand());
+ batchCommands.add(createIndexCommand());
+
+ return batchCommands;
+ }
+
+ private RemoteCommand inject() {
+ RemoteCommandBuilder builder =
RemoteCommandBuilder.instance(JobType.INJECT)
+ .withCrawlId(crawl.getCrawlId()).withArgument("seedDir",
crawl.getSeedDirectory());
+ return builder.build();
+ }
+
+ private RemoteCommand createGenerateCommand() {
+ return createBuilder(JobType.GENERATE).build();
+ }
+
+ private RemoteCommand createFetchCommand() {
+ return
createBuilder(JobType.FETCH).withTimeout(Duration.standardSeconds(50)).build();
+ }
+
+ private RemoteCommand createParseCommand() {
+ return createBuilder(JobType.PARSE).build();
+ }
+
+ private RemoteCommand createIndexCommand() {
+ return createBuilder(JobType.INDEX).build();
+ }
+
+ private RemoteCommand createUpdateDbCommand() {
+ return createBuilder(JobType.UPDATEDB).build();
+ }
+
+ private RemoteCommandBuilder createBuilder(JobType jobType) {
+ return
RemoteCommandBuilder.instance(jobType).withCrawlId(crawl.getCrawlId())
+ .withArgument("batch", batchId);
+ }
+
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/ConnectionStatus.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/ConnectionStatus.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/ConnectionStatus.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/ConnectionStatus.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.model;
+
+public enum ConnectionStatus {
+ CONNECTING, CONNECTED, DISCONNECTED;
+}
\ No newline at end of file
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/Crawl.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/Crawl.java?rev=1626786&view=auto
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/Crawl.java
(added)
+++ nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/Crawl.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.model;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+import org.apache.nutch.webui.model.SeedList;
+
+import com.j256.ormlite.field.DatabaseField;
+
+@Entity
+public class Crawl implements Serializable {
+ public enum CrawlStatus {
+ NEW, CRAWLING, FINISHED, ERROR
+ }
+
+ @Id
+ @GeneratedValue
+ private Long id;
+
+ @Column
+ private String crawlId;
+
+ @Column
+ private String crawlName;
+
+ @Column
+ private CrawlStatus status = CrawlStatus.NEW;
+
+ @Column
+ private Integer numberOfRounds = 1;
+
+ @Column
+ @DatabaseField(foreign = true, foreignAutoRefresh = true)
+ private SeedList seedList;
+
+ @Column
+ private String seedDirectory;
+
+ @Column
+ private int progress;
+
+ public Integer getNumberOfRounds() {
+ return numberOfRounds;
+ }
+
+ public void setNumberOfRounds(Integer numberOfRounds) {
+ this.numberOfRounds = numberOfRounds;
+ }
+
+ public String getCrawlId() {
+ return crawlId;
+ }
+
+ public void setCrawlId(String crawlId) {
+ this.crawlId = crawlId;
+ }
+
+ public CrawlStatus getStatus() {
+ return status;
+ }
+
+ public void setStatus(CrawlStatus status) {
+ this.status = status;
+ }
+
+ public String getCrawlName() {
+ return crawlName;
+ }
+
+ public void setCrawlName(String crawlName) {
+ this.crawlName = crawlName;
+ }
+
+ public SeedList getSeedList() {
+ return seedList;
+ }
+
+ public void setSeedList(SeedList seedList) {
+ this.seedList = seedList;
+ }
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getSeedDirectory() {
+ return seedDirectory;
+ }
+
+ public void setSeedDirectory(String seedDirectory) {
+ this.seedDirectory = seedDirectory;
+ }
+
+ public int getProgress() {
+ return progress;
+ }
+
+ public void setProgress(int progress) {
+ this.progress = progress;
+ }
+
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/JobConfig.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/JobConfig.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/JobConfig.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/JobConfig.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,77 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
******************************************************************************/
+package org.apache.nutch.webui.client.model;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.nutch.webui.client.model.JobInfo.JobType;
+
+import com.google.common.collect.Maps;
+
+public class JobConfig implements Serializable {
+ private String crawlId;
+ private JobType type;
+ private String confId = "default";
+ private String jobClassName;
+ private Map<String, Object> args = Maps.newHashMap();
+
+ public void setArgument(String key, String value) {
+ args.put(key, value);
+ }
+
+ public String getCrawlId() {
+ return crawlId;
+ }
+
+ public void setCrawlId(String crawlId) {
+ this.crawlId = crawlId;
+ }
+
+ public JobType getType() {
+ return type;
+ }
+
+ public void setType(JobType type) {
+ this.type = type;
+ }
+
+ public String getConfId() {
+ return confId;
+ }
+
+ public void setConfId(String confId) {
+ this.confId = confId;
+ }
+
+ public Map<String, Object> getArgs() {
+ return Collections.unmodifiableMap(args);
+ }
+
+ public void setArgs(Map<String, Object> args) {
+ this.args = args;
+ }
+
+ public String getJobClassName() {
+ return jobClassName;
+ }
+
+ public void setJobClassName(String jobClass) {
+ this.jobClassName = jobClass;
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/JobInfo.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/JobInfo.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/JobInfo.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/JobInfo.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,104 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
******************************************************************************/
+package org.apache.nutch.webui.client.model;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class JobInfo implements Serializable {
+ public static enum JobType {
+ INJECT, GENERATE, FETCH, PARSE, UPDATEDB, INDEX, READDB, CLASS
+ };
+
+ public static enum State {
+ IDLE, RUNNING, FINISHED, FAILED, KILLED, STOPPING, KILLING, ANY
+ };
+
+ private String id;
+ private String type;
+ private String confId;
+ private Map<String, Object> args;
+ private Map<String, Object> result;
+ private State state;
+ private String msg;
+ private String crawlId;
+
+ public String getMsg() {
+ return msg;
+ }
+
+ public void setMsg(String msg) {
+ this.msg = msg;
+ }
+
+ public State getState() {
+ return state;
+ }
+
+ public void setState(State state) {
+ this.state = state;
+ }
+
+ public Map<String, Object> getResult() {
+ return result;
+ }
+
+ public void setResult(Map<String, Object> result) {
+ this.result = result;
+ }
+
+ public Map<String, Object> getArgs() {
+ return args;
+ }
+
+ public void setArgs(Map<String, Object> args) {
+ this.args = args;
+ }
+
+ public String getConfId() {
+ return confId;
+ }
+
+ public void setConfId(String confId) {
+ this.confId = confId;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getCrawlId() {
+ return crawlId;
+ }
+
+ public void setCrawlId(String crawlId) {
+ this.crawlId = crawlId;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/NutchStatus.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/NutchStatus.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/NutchStatus.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/client/model/NutchStatus.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,62 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
******************************************************************************/
+package org.apache.nutch.webui.client.model;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Set;
+
+public class NutchStatus implements Serializable {
+
+ private Date startDate;
+ private Set<String> configuration;
+ private Collection<JobInfo> jobs;
+ private Collection<JobInfo> runningJobs;
+
+ public Date getStartDate() {
+ return startDate;
+ }
+
+ public void setStartDate(Date startDate) {
+ this.startDate = startDate;
+ }
+
+ public Set<String> getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Set<String> configuration) {
+ this.configuration = configuration;
+ }
+
+ public Collection<JobInfo> getJobs() {
+ return jobs;
+ }
+
+ public void setJobs(Collection<JobInfo> jobs) {
+ this.jobs = jobs;
+ }
+
+ public Collection<JobInfo> getRunningJobs() {
+ return runningJobs;
+ }
+
+ public void setRunningJobs(Collection<JobInfo> runningJobs) {
+ this.runningJobs = runningJobs;
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/CustomDaoFactory.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/config/CustomDaoFactory.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/CustomDaoFactory.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/CustomDaoFactory.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.config;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.j256.ormlite.dao.Dao;
+import com.j256.ormlite.spring.DaoFactory;
+import com.j256.ormlite.support.ConnectionSource;
+
+public class CustomDaoFactory {
+ private ConnectionSource connectionSource;
+ private List<Dao<?, ?>> registredDaos = Collections.synchronizedList(new
ArrayList<Dao<?, ?>>());
+
+ public CustomDaoFactory(ConnectionSource connectionSource) {
+ this.connectionSource = connectionSource;
+ }
+
+ public <T, ID> Dao<T, ID> createDao(Class<T> clazz) {
+ try {
+ Dao<T, ID> dao = DaoFactory.createDao(connectionSource, clazz);
+ register(dao);
+ return dao;
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private <T, ID> void register(Dao<T, ID> dao) {
+ synchronized (registredDaos) {
+ registredDaos.add(dao);
+ }
+ }
+
+ public List<Dao<?, ?>> getCreatedDaos() {
+ synchronized (registredDaos) {
+ return Collections.unmodifiableList(registredDaos);
+ }
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/CustomTableCreator.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/config/CustomTableCreator.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/CustomTableCreator.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/CustomTableCreator.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.config;
+
+import java.sql.SQLException;
+import java.util.List;
+
+import com.j256.ormlite.dao.BaseDaoImpl;
+import com.j256.ormlite.dao.Dao;
+import com.j256.ormlite.support.ConnectionSource;
+import com.j256.ormlite.table.DatabaseTableConfig;
+import com.j256.ormlite.table.TableUtils;
+
+public class CustomTableCreator {
+
+ private ConnectionSource connectionSource;
+ private List<Dao<?, ?>> configuredDaos;
+
+ public CustomTableCreator(ConnectionSource connectionSource, List<Dao<?, ?>>
configuredDaos) {
+ this.connectionSource = connectionSource;
+ this.configuredDaos = configuredDaos;
+ initialize();
+ }
+
+ private void initialize() {
+ if (configuredDaos == null) {
+ throw new IllegalStateException("configuredDaos was not set in " +
getClass().getSimpleName());
+ }
+
+ for (Dao<?, ?> dao : configuredDaos) {
+ createTableForDao(dao);
+ }
+ }
+
+ private void createTableForDao(Dao<?, ?> dao) {
+ DatabaseTableConfig<?> tableConfig = getTableConfig(dao);
+ createTableIfNotExists(tableConfig);
+ }
+
+ private DatabaseTableConfig<?> getTableConfig(Dao<?, ?> dao) {
+ Class<?> clazz = dao.getDataClass();
+ DatabaseTableConfig<?> tableConfig = null;
+ if (dao instanceof BaseDaoImpl) {
+ tableConfig = ((BaseDaoImpl<?, ?>) dao).getTableConfig();
+ }
+ if (tableConfig == null) {
+ return getConfigFromClass(clazz);
+ }
+ return tableConfig;
+ }
+
+ private DatabaseTableConfig<?> getConfigFromClass(Class<?> clazz) {
+ try {
+ return DatabaseTableConfig.fromClass(connectionSource, clazz);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void createTableIfNotExists(DatabaseTableConfig<?> tableConfig) {
+ try {
+ TableUtils.createTableIfNotExists(connectionSource, tableConfig);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/NutchGuiConfiguration.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/config/NutchGuiConfiguration.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/NutchGuiConfiguration.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/NutchGuiConfiguration.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.config;
+
+import java.util.List;
+
+import org.apache.nutch.webui.model.NutchInstance;
+
+public class NutchGuiConfiguration {
+ private List<NutchInstance> instances;
+
+ public List<NutchInstance> getInstances() {
+ return instances;
+ }
+
+ public void setInstances(List<NutchInstance> instances) {
+ this.instances = instances;
+ }
+}
Added:
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/SpringConfiguration.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/config/SpringConfiguration.java?rev=1626786&view=auto
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/SpringConfiguration.java
(added)
+++
nutch/branches/2.x/src/java/org/apache/nutch/webui/config/SpringConfiguration.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.config;
+
+import java.sql.SQLException;
+import java.util.concurrent.Executor;
+
+import org.apache.nutch.webui.client.model.Crawl;
+import org.apache.nutch.webui.model.NutchInstance;
+import org.apache.nutch.webui.model.SeedList;
+import org.apache.nutch.webui.model.SeedUrl;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.AsyncConfigurer;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+import com.j256.ormlite.dao.Dao;
+import com.j256.ormlite.db.H2DatabaseType;
+import com.j256.ormlite.jdbc.JdbcConnectionSource;
+
+@Configuration
+@EnableAsync
+public class SpringConfiguration implements AsyncConfigurer {
+
+ @Override
+ public Executor getAsyncExecutor() {
+ // TODO move magic numbers to properties file
+ ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+ executor.setCorePoolSize(7);
+ executor.setMaxPoolSize(42);
+ executor.setQueueCapacity(11);
+ executor.setThreadNamePrefix("SpringExecutor-");
+ executor.initialize();
+ return executor;
+ }
+
+ @Bean
+ public JdbcConnectionSource getConnectionSource() throws SQLException {
+ JdbcConnectionSource source = new
JdbcConnectionSource("jdbc:h2:~/.nutch/config",
+ new H2DatabaseType());
+ source.initialize();
+ return source;
+ }
+
+ @Bean
+ public CustomDaoFactory getDaoFactory() throws SQLException {
+ return new CustomDaoFactory(getConnectionSource());
+ }
+
+ @Bean
+ public Dao<NutchInstance, Long> createNutchDao() throws SQLException {
+ return getDaoFactory().createDao(NutchInstance.class);
+ }
+
+ @Bean
+ public Dao<SeedList, Long> createSeedListDao() throws SQLException {
+ return getDaoFactory().createDao(SeedList.class);
+ }
+
+ @Bean
+ public Dao<SeedUrl, Long> createSeedUrlDao() throws SQLException {
+ return getDaoFactory().createDao(SeedUrl.class);
+ }
+
+ @Bean
+ public Dao<Crawl, Long> createCrawlDao() throws SQLException {
+ return getDaoFactory().createDao(Crawl.class);
+ }
+
+ @Bean
+ public CustomTableCreator createTableCreator() throws SQLException {
+ return new CustomTableCreator(getConnectionSource(),
getDaoFactory().getCreatedDaos());
+ }
+
+}
Added: nutch/branches/2.x/src/java/org/apache/nutch/webui/model/NutchConfig.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/webui/model/NutchConfig.java?rev=1626786&view=auto
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/webui/model/NutchConfig.java
(added)
+++ nutch/branches/2.x/src/java/org/apache/nutch/webui/model/NutchConfig.java
Mon Sep 22 14:18:53 2014
@@ -0,0 +1,19 @@
+package org.apache.nutch.webui.model;
+
+public class NutchConfig {
+ private String name = "name";
+ private String value;
+
+ public void setName (String name){
+ this.name = name;
+ }
+ public String getName(){
+ return this.name;
+ }
+ public String getValue() {
+ return value;
+ }
+ public void setValue(String value) {
+ this.value = value;
+ }
+}
