Author: lewismc
Date: Mon Sep 28 18:58:33 2015
New Revision: 1705744
URL: http://svn.apache.org/viewvc?rev=1705744&view=rev
Log:
NUTCH-2086 Nutch 1.X Webui this closes #61
Added:
nutch/trunk/src/java/org/apache/nutch/webui/
nutch/trunk/src/java/org/apache/nutch/webui/NutchUiApplication.java
nutch/trunk/src/java/org/apache/nutch/webui/NutchUiApplication.properties
nutch/trunk/src/java/org/apache/nutch/webui/NutchUiServer.java
nutch/trunk/src/java/org/apache/nutch/webui/client/
nutch/trunk/src/java/org/apache/nutch/webui/client/NutchClient.java
nutch/trunk/src/java/org/apache/nutch/webui/client/NutchClientFactory.java
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/CrawlingCycle.java
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/CrawlingCycleListener.java
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/NutchClientImpl.java
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommand.java
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandBuilder.java
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandExecutor.java
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandsBatchFactory.java
nutch/trunk/src/java/org/apache/nutch/webui/client/model/
nutch/trunk/src/java/org/apache/nutch/webui/client/model/ConnectionStatus.java
nutch/trunk/src/java/org/apache/nutch/webui/client/model/Crawl.java
nutch/trunk/src/java/org/apache/nutch/webui/client/model/JobConfig.java
nutch/trunk/src/java/org/apache/nutch/webui/client/model/JobInfo.java
nutch/trunk/src/java/org/apache/nutch/webui/client/model/NutchStatus.java
nutch/trunk/src/java/org/apache/nutch/webui/config/
nutch/trunk/src/java/org/apache/nutch/webui/config/CustomDaoFactory.java
nutch/trunk/src/java/org/apache/nutch/webui/config/CustomTableCreator.java
nutch/trunk/src/java/org/apache/nutch/webui/config/NutchGuiConfiguration.java
nutch/trunk/src/java/org/apache/nutch/webui/config/SpringConfiguration.java
nutch/trunk/src/java/org/apache/nutch/webui/model/
nutch/trunk/src/java/org/apache/nutch/webui/model/NutchConfig.java
nutch/trunk/src/java/org/apache/nutch/webui/model/NutchInstance.java
nutch/trunk/src/java/org/apache/nutch/webui/model/SeedList.java
nutch/trunk/src/java/org/apache/nutch/webui/model/SeedUrl.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/
nutch/trunk/src/java/org/apache/nutch/webui/pages/AbstractBasePage.html
nutch/trunk/src/java/org/apache/nutch/webui/pages/AbstractBasePage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/DashboardPage.html
nutch/trunk/src/java/org/apache/nutch/webui/pages/DashboardPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/LogOutPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/SchedulingPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/SearchPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/StatisticsPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/UrlsUploadPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/UserSettingsPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/assets/
nutch/trunk/src/java/org/apache/nutch/webui/pages/assets/NutchUiCssReference.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/assets/nutch-style.css
nutch/trunk/src/java/org/apache/nutch/webui/pages/components/
nutch/trunk/src/java/org/apache/nutch/webui/pages/components/ColorEnumLabel.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/components/ColorEnumLabelBuilder.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/components/CpmIteratorAdapter.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/crawls/
nutch/trunk/src/java/org/apache/nutch/webui/pages/crawls/CrawlPanel.html
nutch/trunk/src/java/org/apache/nutch/webui/pages/crawls/CrawlPanel.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/crawls/CrawlsPage.html
nutch/trunk/src/java/org/apache/nutch/webui/pages/crawls/CrawlsPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/instances/
nutch/trunk/src/java/org/apache/nutch/webui/pages/instances/InstancePanel.html
nutch/trunk/src/java/org/apache/nutch/webui/pages/instances/InstancePanel.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/instances/InstancesPage.html
nutch/trunk/src/java/org/apache/nutch/webui/pages/instances/InstancesPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/menu/
nutch/trunk/src/java/org/apache/nutch/webui/pages/menu/VerticalMenu.html
nutch/trunk/src/java/org/apache/nutch/webui/pages/menu/VerticalMenu.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/seed/
nutch/trunk/src/java/org/apache/nutch/webui/pages/seed/SeedListsPage.html
nutch/trunk/src/java/org/apache/nutch/webui/pages/seed/SeedListsPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/seed/SeedPage.html
nutch/trunk/src/java/org/apache/nutch/webui/pages/seed/SeedPage.java
nutch/trunk/src/java/org/apache/nutch/webui/pages/settings/
nutch/trunk/src/java/org/apache/nutch/webui/pages/settings/SettingsPage.html
nutch/trunk/src/java/org/apache/nutch/webui/pages/settings/SettingsPage.java
nutch/trunk/src/java/org/apache/nutch/webui/service/
nutch/trunk/src/java/org/apache/nutch/webui/service/CrawlService.java
nutch/trunk/src/java/org/apache/nutch/webui/service/NutchInstanceService.java
nutch/trunk/src/java/org/apache/nutch/webui/service/NutchService.java
nutch/trunk/src/java/org/apache/nutch/webui/service/SeedListService.java
nutch/trunk/src/java/org/apache/nutch/webui/service/impl/
nutch/trunk/src/java/org/apache/nutch/webui/service/impl/CrawlServiceImpl.java
nutch/trunk/src/java/org/apache/nutch/webui/service/impl/NutchInstanceServiceImpl.java
nutch/trunk/src/java/org/apache/nutch/webui/service/impl/NutchServiceImpl.java
nutch/trunk/src/java/org/apache/nutch/webui/service/impl/SeedListServiceImpl.java
Modified:
nutch/trunk/ivy/ivy.xml
nutch/trunk/src/bin/nutch
nutch/trunk/src/java/org/apache/nutch/service/model/request/SeedList.java
nutch/trunk/src/java/org/apache/nutch/service/model/request/SeedUrl.java
Modified: nutch/trunk/ivy/ivy.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/ivy/ivy.xml?rev=1705744&r1=1705743&r2=1705744&view=diff
==============================================================================
--- nutch/trunk/ivy/ivy.xml (original)
+++ nutch/trunk/ivy/ivy.xml Mon Sep 28 18:58:33 2015
@@ -91,17 +91,28 @@
<!--artifacts needed for testing -->
<dependency org="junit" name="junit" rev="4.11"
conf="test->default" />
- <!--dependency org="org.apache.hadoop" name="hadoop-test"
rev="1.2.0" conf="test->default" /-->
<dependency org="org.mortbay.jetty" name="jetty-client"
rev="6.1.22" conf="test->default" />
<dependency org="org.mortbay.jetty" name="jetty" rev="6.1.22"
conf="test->default" />
<dependency org="org.mortbay.jetty" name="jetty-util"
rev="6.1.22" conf="test->default" />
<!-- end of test artifacts -->
+ <!-- web app dependencies -->
- <dependency org="org.mortbay.jetty" name="jetty-client"
rev="6.1.22" conf="test->default" />
+ <dependency org="org.apache.commons" name="commons-collections4"
rev="4.0" conf="*->default" />
+ <dependency org="org.springframework" name="spring-core"
rev="4.0.4.RELEASE" conf="*->default" />
+ <dependency org="org.springframework" name="spring-context"
rev="4.0.4.RELEASE" conf="*->default" />
+ <dependency org="org.springframework" name="spring-web"
rev="4.0.4.RELEASE" conf="*->default" />
- <dependency org="org.mortbay.jetty" name="jetty" rev="6.1.22"
conf="test->default" />
- <dependency org="org.mortbay.jetty" name="jetty-util"
rev="6.1.22" conf="test->default" />
+ <dependency org="com.sun.jersey" name="jersey-client" rev="1.8"
conf="*->default" />
+
+ <dependency org="com.j256.ormlite" name="ormlite-jdbc" rev="4.48"
conf="*->default" />
+ <dependency org="com.h2database" name="h2" rev="1.4.180"
conf="*->default" />
+ <dependency org="org.eclipse.persistence" name="javax.persistence"
rev="2.0.0" conf="*->default" />
+
+ <dependency org="org.apache.wicket" name="wicket-core" rev="6.16.0"
conf="*->default" />
+ <dependency org="org.apache.wicket" name="wicket-spring" rev="6.16.0"
conf="*->default" />
+ <dependency org="de.agilecoders.wicket" name="wicket-bootstrap-core"
rev="0.9.2" conf="*->default" />
+ <dependency org="de.agilecoders.wicket"
name="wicket-bootstrap-extensions" rev="0.9.2" conf="*->default" />
<!--global exclusion -->
<exclude module="jmxtools" />
Modified: nutch/trunk/src/bin/nutch
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/bin/nutch?rev=1705744&r1=1705743&r2=1705744&view=diff
==============================================================================
--- nutch/trunk/src/bin/nutch (original)
+++ nutch/trunk/src/bin/nutch Mon Sep 28 18:58:33 2015
@@ -86,8 +86,9 @@ if [ $# = 0 ]; then
echo " nodedumper dumps the web graph's node scores"
echo " plugin load a plugin and run one of its classes main()"
echo " junit runs the given JUnit test"
- echo " startserver runs the Nutch Server on localhost:8081"
- echo " warc exports crawled data from segments at the WARC
format"
+ echo " startserver runs the Nutch Server on localhost:8081"
+ echo " webapp run a local Nutch Web Application on locahost:8080"
+ echo " warc exports crawled data from segments at the WARC
format"
echo " or"
echo " CLASSNAME run the class named CLASSNAME"
echo "Most commands print help when invoked w/o parameters."
@@ -279,6 +280,8 @@ elif [ "$COMMAND" = "junit" ] ; then
CLASS=org.junit.runner.JUnitCore
elif [ "$COMMAND" = "startserver" ] ; then
CLASS=org.apache.nutch.service.NutchServer
+elif [ "$COMMAND" = "webapp" ] ; then
+ CLASS=org.apache.nutch.webui.NutchUiServer
elif [ "$COMMAND" = "warc" ] ; then
CLASS=org.apache.nutch.tools.warc.WARCExporter
else
Modified:
nutch/trunk/src/java/org/apache/nutch/service/model/request/SeedList.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/model/request/SeedList.java?rev=1705744&r1=1705743&r2=1705744&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/model/request/SeedList.java
(original)
+++ nutch/trunk/src/java/org/apache/nutch/service/model/request/SeedList.java
Mon Sep 28 18:58:33 2015
@@ -19,12 +19,18 @@ package org.apache.nutch.service.model.r
import java.io.Serializable;
import java.util.Collection;
+import org.apache.commons.collections4.CollectionUtils;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonManagedReference;
+
public class SeedList implements Serializable {
private Long id;
private String name;
+ @JsonManagedReference
private Collection<SeedUrl> seedUrls;
public Long getId() {
@@ -51,6 +57,14 @@ public class SeedList implements Seriali
this.name = name;
}
+ @JsonIgnore
+ public int getSeedUrlsCount() {
+ if (CollectionUtils.isEmpty(seedUrls)) {
+ return 0;
+ }
+ return seedUrls.size();
+ }
+
@Override
public int hashCode() {
final int prime = 31;
Modified:
nutch/trunk/src/java/org/apache/nutch/service/model/request/SeedUrl.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/model/request/SeedUrl.java?rev=1705744&r1=1705743&r2=1705744&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/model/request/SeedUrl.java
(original)
+++ nutch/trunk/src/java/org/apache/nutch/service/model/request/SeedUrl.java
Mon Sep 28 18:58:33 2015
@@ -18,10 +18,14 @@ package org.apache.nutch.service.model.r
import java.io.Serializable;
+import com.fasterxml.jackson.annotation.JsonBackReference;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
public class SeedUrl implements Serializable {
private Long id;
+ @JsonBackReference
private SeedList seedList;
private String url;
@@ -48,10 +52,12 @@ public class SeedUrl implements Serializ
this.url = url;
}
+ @JsonIgnore
public SeedList getSeedList() {
return seedList;
}
+ @JsonIgnore
public void setSeedList(SeedList seedList) {
this.seedList = seedList;
}
Added: nutch/trunk/src/java/org/apache/nutch/webui/NutchUiApplication.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/NutchUiApplication.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/NutchUiApplication.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/NutchUiApplication.java Mon Sep
28 18:58:33 2015
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui;
+
+import org.apache.nutch.webui.pages.DashboardPage;
+import org.apache.nutch.webui.pages.assets.NutchUiCssReference;
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.protocol.http.WebApplication;
+import org.apache.wicket.spring.injection.annot.SpringComponentInjector;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Component;
+
+import de.agilecoders.wicket.core.Bootstrap;
+import
de.agilecoders.wicket.core.markup.html.themes.bootstrap.BootstrapCssReference;
+import de.agilecoders.wicket.core.settings.BootstrapSettings;
+import de.agilecoders.wicket.core.settings.SingleThemeProvider;
+import de.agilecoders.wicket.core.settings.Theme;
+import
de.agilecoders.wicket.extensions.markup.html.bootstrap.icon.FontAwesomeCssReference;
+
+@Component
+public class NutchUiApplication extends WebApplication implements
+ ApplicationContextAware {
+ private static final String THEME_NAME = "bootstrap";
+ private ApplicationContext context;
+
+ /**
+ * @see org.apache.wicket.Application#getHomePage()
+ */
+ @Override
+ public Class<? extends WebPage> getHomePage() {
+ return DashboardPage.class;
+ }
+
+ /**
+ * @see org.apache.wicket.Application#init()
+ */
+ @Override
+ public void init() {
+ super.init();
+ BootstrapSettings settings = new BootstrapSettings();
+ Bootstrap.install(this, settings);
+ configureTheme(settings);
+
+ getComponentInstantiationListeners().add(
+ new SpringComponentInjector(this, context));
+ }
+
+ private void configureTheme(BootstrapSettings settings) {
+ Theme theme = new Theme(THEME_NAME, BootstrapCssReference.instance(),
+ FontAwesomeCssReference.instance(), NutchUiCssReference.instance());
+ settings.setThemeProvider(new SingleThemeProvider(theme));
+ }
+
+ @Override
+ public void setApplicationContext(ApplicationContext applicationContext)
+ throws BeansException {
+ this.context = applicationContext;
+ }
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/NutchUiApplication.properties
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/NutchUiApplication.properties?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/NutchUiApplication.properties
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/NutchUiApplication.properties
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,63 @@
+#############################################################################
+#Licensed to the Apache Software Foundation (ASF) under one or more
+#contributor license agreements. See the NOTICE file distributed with
+#this work for additional information regarding copyright ownership.
+#The ASF licenses this file to You under the Apache License, Version 2.0
+#(the "License"); you may not use this file except in compliance with
+#the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#############################################################################
+
+navbar.menu.dashboard = Dashboard
+navbar.menu.statistics = Statistics
+navbar.menu.instances = Instances
+navbar.menu.settings = Settings
+navbar.menu.crawls = Crawls
+navbar.menu.scheduling = Scheduling
+navbar.menu.search = Search
+navbar.menu.url = URLs upload
+navbar.menu.seedLists = Seed lists
+
+page.header.seedList = Seed list
+
+navbar.userMenu.settings = Settings
+navbar.userMenu.logout = Log out
+
+menu.settings=Settings
+menu.instances=Instances
+
+connected=Connected
+disconnected=Disconnected
+
+##ENUMS
+ConnectionStatus.CONNECTING=Connecting
+ConnectionStatus.CONNECTED=Connected
+ConnectionStatus.DISCONNECTED=Disconnected
+
+CrawlStatus.NEW=New
+CrawlStatus.ERROR=Error
+CrawlStatus.CRAWLING=Crawling
+CrawlStatus.FINISHED=Finished
+
+instances=Instances
+instances.header.name=Instance name
+instances.header.hostname=Hostname
+instances.header.status=Status
+instances.header.username=Username
+instances.label.name=Instance name
+instances.label.hostname=Hostname
+instances.label.port=Port
+instances.label.username=Username
+instances.label.password=Password
+instances.buttons.addInstance=Add instance
+
+settings=Settings
+settings.header.name = Name
+settings.header.value = Value
\ No newline at end of file
Added: nutch/trunk/src/java/org/apache/nutch/webui/NutchUiServer.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/NutchUiServer.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/NutchUiServer.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/NutchUiServer.java Mon Sep 28
18:58:33 2015
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.wicket.protocol.http.WicketFilter;
+import org.apache.wicket.spring.SpringWebApplicationFactory;
+import org.mortbay.jetty.Handler;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.servlet.Context;
+import org.mortbay.jetty.servlet.DefaultServlet;
+import org.mortbay.jetty.servlet.FilterHolder;
+import org.springframework.web.context.ContextLoaderListener;
+import org.springframework.web.context.WebApplicationContext;
+import org.springframework.web.context.request.RequestContextListener;
+import
org.springframework.web.context.support.AnnotationConfigWebApplicationContext;
+
+public class NutchUiServer {
+ private static final String APP_FACTORY_NAME =
SpringWebApplicationFactory.class
+ .getName();
+ private static final String CONFIG_LOCATION = "org.apache.nutch.webui";
+ private static final String CMD_PORT = "port";
+ private static Integer port = 8080;
+
+ public static void main(String[] args) throws Exception {
+ CommandLineParser parser = new GnuParser();
+ Options options = createWebAppOptions();
+ CommandLine commandLine = null;
+ HelpFormatter formatter = new HelpFormatter();
+ try {
+ commandLine = parser.parse(options, args);
+ } catch (Exception e) {
+ formatter.printHelp("NutchUiServer", options, true);
+ StringUtils.stringifyException(e);
+ }
+
+ if (commandLine.hasOption("help")) {
+ formatter.printHelp("NutchUiServer", options, true);
+ return;
+ }
+ if (commandLine.hasOption(CMD_PORT)) {
+ port = Integer.parseInt(commandLine.getOptionValue(CMD_PORT));
+ }
+ startServer();
+ }
+
+ private static void startServer() throws Exception, InterruptedException {
+ Server server = new Server(port);
+ Context context = new Context(server, "/", Context.SESSIONS);
+ context.addServlet(DefaultServlet.class, "/*");
+
+ context.addEventListener(new ContextLoaderListener(getContext()));
+ context.addEventListener(new RequestContextListener());
+
+ WicketFilter filter = new WicketFilter();
+ filter.setFilterPath("/");
+ FilterHolder holder = new FilterHolder(filter);
+ holder.setInitParameter("applicationFactoryClassName", APP_FACTORY_NAME);
+ context.addFilter(holder, "/*", Handler.DEFAULT);
+
+ server.setHandler(context);
+ server.start();
+ server.join();
+ }
+
+ private static WebApplicationContext getContext() {
+ AnnotationConfigWebApplicationContext context = new
AnnotationConfigWebApplicationContext();
+ context.setConfigLocation(CONFIG_LOCATION);
+ return context;
+ }
+
+ private static Options createWebAppOptions() {
+ Options options = new Options();
+ Option helpOpt = new Option("h", "help", false, "show this help message");
+ OptionBuilder.withDescription("Port to run the WebApplication on.");
+ OptionBuilder.hasOptionalArg();
+ OptionBuilder.withArgName("port number");
+ options.addOption(OptionBuilder.create(CMD_PORT));
+ options.addOption(helpOpt);
+ return options;
+ }
+
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/client/NutchClient.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/NutchClient.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/client/NutchClient.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/client/NutchClient.java Mon Sep
28 18:58:33 2015
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client;
+
+import java.util.Map;
+
+import org.apache.nutch.webui.client.model.ConnectionStatus;
+import org.apache.nutch.webui.client.model.JobConfig;
+import org.apache.nutch.webui.client.model.JobInfo;
+import org.apache.nutch.webui.client.model.NutchStatus;
+import org.apache.nutch.webui.model.NutchInstance;
+import org.apache.nutch.webui.model.SeedList;
+
+public interface NutchClient {
+
+ public NutchInstance getNutchInstance();
+
+ public NutchStatus getNutchStatus();
+
+ public ConnectionStatus getConnectionStatus();
+
+ public String executeJob(JobConfig jobConfig);
+
+ public JobInfo getJobInfo(String jobId);
+
+ public Map<String, String> getNutchConfig(String config);
+
+ /**
+ * Create seed list and return seed directory location
+ *
+ * @param seedList
+ * @return
+ */
+ public String createSeed(SeedList seedList);
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/client/NutchClientFactory.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/NutchClientFactory.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/client/NutchClientFactory.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/client/NutchClientFactory.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client;
+
+import java.util.concurrent.ExecutionException;
+
+import org.apache.nutch.webui.client.impl.NutchClientImpl;
+import org.apache.nutch.webui.model.NutchInstance;
+import org.springframework.stereotype.Component;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+@Component
+public class NutchClientFactory {
+ private LoadingCache<NutchInstance, NutchClient> cache;
+
+ public NutchClientFactory() {
+ cache = CacheBuilder.newBuilder().build(new NutchClientCacheLoader());
+ }
+
+ public NutchClient getClient(NutchInstance instance) {
+ try {
+ return cache.get(instance);
+ } catch (ExecutionException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private static class NutchClientCacheLoader extends
+ CacheLoader<NutchInstance, NutchClient> {
+ @Override
+ public NutchClient load(NutchInstance key) throws Exception {
+ return new NutchClientImpl(key);
+ }
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/CrawlingCycle.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/impl/CrawlingCycle.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/client/impl/CrawlingCycle.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/client/impl/CrawlingCycle.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import java.util.List;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.nutch.webui.client.model.Crawl;
+import org.apache.nutch.webui.client.model.JobInfo;
+import org.apache.nutch.webui.client.model.JobInfo.State;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+/**
+ * This class implements crawl cycle as in crawl script
+ *
+ * @author feodor
+ *
+ */
+public class CrawlingCycle {
+ private Logger log = LoggerFactory.getLogger(CrawlingCycle.class);
+
+ private CrawlingCycleListener listener;
+ private RemoteCommandExecutor executor;
+ private Crawl crawl;
+
+ private List<RemoteCommand> remoteCommands;
+ private List<RemoteCommand> executedCommands = Lists.newArrayList();
+
+ public CrawlingCycle(CrawlingCycleListener listener,
+ RemoteCommandExecutor executor, Crawl crawl, List<RemoteCommand>
commands) {
+ this.listener = listener;
+ this.executor = executor;
+ this.crawl = crawl;
+ this.remoteCommands = commands;
+ }
+
+ public synchronized void executeCrawlCycle() {
+ listener.crawlingStarted(crawl);
+
+ for (RemoteCommand command : remoteCommands) {
+ JobInfo jobInfo = executor.executeRemoteJob(command);
+ command.setJobInfo(jobInfo);
+
+ log.info("Executed remote command data: {}", command);
+
+ if (jobInfo.getState() == State.FAILED) {
+ listener.onCrawlError(crawl, jobInfo.getMsg());
+ return;
+ }
+
+ executedCommands.add(command);
+ listener.commandExecuted(crawl, command, calculateProgress());
+ }
+ listener.crawlingFinished(crawl);
+ }
+
+ private int calculateProgress() {
+ if (CollectionUtils.isEmpty(remoteCommands)) {
+ return 0;
+ }
+ return (int) ((float) executedCommands.size()
+ / (float) remoteCommands.size() * 100);
+ }
+
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/CrawlingCycleListener.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/impl/CrawlingCycleListener.java?rev=1705744&view=auto
==============================================================================
---
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/CrawlingCycleListener.java
(added)
+++
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/CrawlingCycleListener.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import org.apache.nutch.webui.client.model.Crawl;
+
+public interface CrawlingCycleListener {
+
+ void crawlingStarted(Crawl crawl);
+
+ void onCrawlError(Crawl crawl, String msg);
+
+ void commandExecuted(Crawl crawl, RemoteCommand command, int progress);
+
+ void crawlingFinished(Crawl crawl);
+
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/NutchClientImpl.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/impl/NutchClientImpl.java?rev=1705744&view=auto
==============================================================================
---
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/NutchClientImpl.java
(added)
+++
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/NutchClientImpl.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+
+import java.util.Map;
+
+import org.apache.nutch.webui.client.NutchClient;
+import org.apache.nutch.webui.client.model.ConnectionStatus;
+import org.apache.nutch.webui.client.model.JobConfig;
+import org.apache.nutch.webui.client.model.JobInfo;
+import org.apache.nutch.webui.client.model.NutchStatus;
+import org.apache.nutch.webui.model.NutchInstance;
+import org.apache.nutch.webui.model.SeedList;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+
+public class NutchClientImpl implements NutchClient {
+ private Client client;
+ private WebResource nutchResource;
+ private NutchInstance instance;
+
+ public NutchClientImpl(NutchInstance instance) {
+ this.instance = instance;
+ createClient();
+ }
+
+ public void createClient() {
+ ClientConfig clientConfig = new DefaultClientConfig();
+ clientConfig.getFeatures()
+ .put(JSONConfiguration.FEATURE_POJO_MAPPING, true);
+ this.client = Client.create(clientConfig);
+ this.nutchResource = client.resource(instance.getUrl());
+ }
+
+ @Override
+ public NutchStatus getNutchStatus() {
+ return nutchResource.path("/admin").type(APPLICATION_JSON)
+ .get(NutchStatus.class);
+ }
+
+ @Override
+ public ConnectionStatus getConnectionStatus() {
+
+ getNutchStatus();
+ return ConnectionStatus.CONNECTED;
+ // TODO implement disconnected status
+ }
+
+ @Override
+ public String executeJob(JobConfig jobConfig) {
+ JobInfo jobInfo = nutchResource.path("/job/create").type(APPLICATION_JSON)
+ .post(JobInfo.class, jobConfig);
+ return jobInfo.getId();
+ }
+
+ @Override
+ public JobInfo getJobInfo(String jobId) {
+ return nutchResource.path("/job/" + jobId).type(APPLICATION_JSON)
+ .get(JobInfo.class);
+ }
+
+ @Override
+ public NutchInstance getNutchInstance() {
+ return instance;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map<String, String> getNutchConfig(String config) {
+ return nutchResource.path("/config/" + config).type(APPLICATION_JSON)
+ .get(Map.class);
+ }
+
+ @Override
+ public String createSeed(SeedList seedList) {
+ return nutchResource.path("/seed/create").type(APPLICATION_JSON)
+ .post(String.class, seedList);
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommand.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommand.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommand.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommand.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import java.io.Serializable;
+import java.text.MessageFormat;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nutch.webui.client.model.JobConfig;
+import org.apache.nutch.webui.client.model.JobInfo;
+import org.joda.time.Duration;
+
+public class RemoteCommand implements Serializable {
+ private JobConfig jobConfig;
+ private JobInfo jobInfo = new JobInfo();
+ private Duration timeout;
+
+ /**
+ * Use {@link RemoteCommandBuilder} instead
+ */
+ @SuppressWarnings("unused")
+ private RemoteCommand() {
+ }
+
+ public RemoteCommand(JobConfig jobConfig) {
+ this.jobConfig = jobConfig;
+ }
+
+ public JobConfig getJobConfig() {
+ return jobConfig;
+ }
+
+ public void setJobConfig(JobConfig jobConfig) {
+ this.jobConfig = jobConfig;
+ }
+
+ public JobInfo getJobInfo() {
+ return jobInfo;
+ }
+
+ public void setJobInfo(JobInfo jobInfo) {
+ this.jobInfo = jobInfo;
+ }
+
+ public Duration getTimeout() {
+ return timeout;
+ }
+
+ public void setTimeout(Duration timeout) {
+ this.timeout = timeout;
+ }
+
+ @Override
+ public String toString() {
+ String statusInfo = StringUtils.EMPTY;
+ if (jobInfo != null) {
+ statusInfo = MessageFormat.format("{0}", jobInfo.getState());
+ }
+ return MessageFormat.format("{0} status: {1}", jobConfig.getType(),
+ statusInfo);
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandBuilder.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandBuilder.java?rev=1705744&view=auto
==============================================================================
---
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandBuilder.java
(added)
+++
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandBuilder.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import org.apache.nutch.webui.client.model.JobConfig;
+import org.apache.nutch.webui.client.model.JobInfo.JobType;
+import org.joda.time.Duration;
+
+public class RemoteCommandBuilder {
+ private JobConfig jobConfig = new JobConfig();
+ private Duration timeout = Duration.standardSeconds(10);
+
+ private RemoteCommandBuilder() {
+ }
+
+ public static RemoteCommandBuilder instance(JobType jobType) {
+ return new RemoteCommandBuilder().withJobType(jobType);
+ }
+
+ public RemoteCommandBuilder withJobType(JobType jobType) {
+ jobConfig.setType(jobType);
+ return this;
+ }
+
+ public RemoteCommandBuilder withConfigId(String configId) {
+ jobConfig.setConfId(configId);
+ return this;
+ }
+
+ public RemoteCommandBuilder withCrawlId(String crawlId) {
+ jobConfig.setCrawlId(crawlId);
+ return this;
+ }
+
+ public RemoteCommandBuilder withArgument(String key, String value) {
+ jobConfig.setArgument(key, value);
+ return this;
+ }
+
+ public RemoteCommandBuilder withTimeout(Duration timeout) {
+ this.timeout = timeout;
+ return this;
+ }
+
+ public RemoteCommand build() {
+ RemoteCommand remoteCommand = new RemoteCommand(jobConfig);
+ remoteCommand.setTimeout(timeout);
+ return remoteCommand;
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandExecutor.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandExecutor.java?rev=1705744&view=auto
==============================================================================
---
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandExecutor.java
(added)
+++
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandExecutor.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.nutch.webui.client.NutchClient;
+import org.apache.nutch.webui.client.model.JobInfo;
+import org.apache.nutch.webui.client.model.JobInfo.State;
+import org.joda.time.DateTimeConstants;
+import org.joda.time.Duration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class executes remote job and waits for success/failure result
+ *
+ * @author feodor
+ *
+ */
+public class RemoteCommandExecutor {
+ private Logger log = LoggerFactory.getLogger(RemoteCommandExecutor.class);
+
+ private static final int DEFAULT_TIMEOUT_SEC = 60;
+ private Duration requestDelay = new Duration(500);
+
+ private NutchClient client;
+ private ExecutorService executor;
+
+ public RemoteCommandExecutor(NutchClient client) {
+ this.client = client;
+ this.executor = Executors.newSingleThreadExecutor();
+ }
+
+ public JobInfo executeRemoteJob(RemoteCommand command) {
+ try {
+ String jobId = client.executeJob(command.getJobConfig());
+ Future<JobInfo> chekerFuture = executor
+ .submit(new JobStateChecker(jobId));
+ return chekerFuture.get(getTimeout(command), TimeUnit.MILLISECONDS);
+ } catch (Exception e) {
+ log.error("Remote command failed", e);
+ JobInfo jobInfo = new JobInfo();
+ jobInfo.setState(State.FAILED);
+ jobInfo.setMsg(ExceptionUtils.getStackTrace(e));
+ return jobInfo;
+ }
+ }
+
+ private long getTimeout(RemoteCommand command) {
+ if (command.getTimeout() == null) {
+ return DEFAULT_TIMEOUT_SEC * DateTimeConstants.MILLIS_PER_SECOND;
+ }
+ return command.getTimeout().getMillis();
+ }
+
+ public void setRequestDelay(Duration requestDelay) {
+ this.requestDelay = requestDelay;
+ }
+
+ public class JobStateChecker implements Callable<JobInfo> {
+
+ private String jobId;
+
+ public JobStateChecker(String jobId) {
+ this.jobId = jobId;
+ }
+
+ @Override
+ public JobInfo call() throws Exception {
+ while (!Thread.interrupted()) {
+ JobInfo jobInfo = client.getJobInfo(jobId);
+ checkState(jobInfo != null, "Cannot get job info!");
+
+ State state = jobInfo.getState();
+ checkState(state != null, "Unknown job state!");
+
+ if (state == State.RUNNING || state == State.ANY || state ==
State.IDLE) {
+ Thread.sleep(requestDelay.getMillis());
+ continue;
+ }
+
+ return jobInfo;
+ }
+ return null;
+ }
+
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandsBatchFactory.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandsBatchFactory.java?rev=1705744&view=auto
==============================================================================
---
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandsBatchFactory.java
(added)
+++
nutch/trunk/src/java/org/apache/nutch/webui/client/impl/RemoteCommandsBatchFactory.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.impl;
+
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.nutch.webui.client.model.Crawl;
+import org.apache.nutch.webui.client.model.JobInfo.JobType;
+import org.joda.time.Duration;
+import org.springframework.beans.factory.config.BeanDefinition;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import com.google.common.collect.Lists;
+
+@Component
+@Scope(BeanDefinition.SCOPE_PROTOTYPE)
+public class RemoteCommandsBatchFactory {
+
+ private List<RemoteCommand> remoteCommands;
+ private Crawl crawl;
+
+ private String batchId;
+
+ public List<RemoteCommand> createCommands(Crawl crawl) {
+ this.crawl = crawl;
+ this.remoteCommands = Lists.newArrayList();
+
+ remoteCommands.add(inject());
+ for (int i = 0; i < crawl.getNumberOfRounds(); i++) {
+ remoteCommands.addAll(createBatchCommands());
+ }
+ return remoteCommands;
+ }
+
+ private List<RemoteCommand> createBatchCommands() {
+ this.batchId = UUID.randomUUID().toString();
+ List<RemoteCommand> batchCommands = Lists.newArrayList();
+
+ batchCommands.add(createGenerateCommand());
+ batchCommands.add(createFetchCommand());
+ batchCommands.add(createParseCommand());
+ batchCommands.add(createUpdateDbCommand());
+ batchCommands.add(createIndexCommand());
+
+ return batchCommands;
+ }
+
+ private RemoteCommand inject() {
+ RemoteCommandBuilder builder = RemoteCommandBuilder
+ .instance(JobType.INJECT).withCrawlId(crawl.getCrawlId())
+ .withArgument("url_dir", crawl.getSeedDirectory());
+ return builder.build();
+ }
+
+ private RemoteCommand createGenerateCommand() {
+ return createBuilder(JobType.GENERATE).build();
+ }
+
+ private RemoteCommand createFetchCommand() {
+ return createBuilder(JobType.FETCH).withTimeout(
+ Duration.standardSeconds(50)).build();
+ }
+
+ private RemoteCommand createParseCommand() {
+ return createBuilder(JobType.PARSE).build();
+ }
+
+ private RemoteCommand createIndexCommand() {
+ return createBuilder(JobType.INDEX).build();
+ }
+
+ private RemoteCommand createUpdateDbCommand() {
+ return createBuilder(JobType.UPDATEDB).build();
+ }
+
+ private RemoteCommandBuilder createBuilder(JobType jobType) {
+ return RemoteCommandBuilder.instance(jobType)
+ .withCrawlId(crawl.getCrawlId()).withArgument("batch", batchId);
+ }
+
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/client/model/ConnectionStatus.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/model/ConnectionStatus.java?rev=1705744&view=auto
==============================================================================
---
nutch/trunk/src/java/org/apache/nutch/webui/client/model/ConnectionStatus.java
(added)
+++
nutch/trunk/src/java/org/apache/nutch/webui/client/model/ConnectionStatus.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.model;
+
+public enum ConnectionStatus {
+ CONNECTING, CONNECTED, DISCONNECTED;
+}
\ No newline at end of file
Added: nutch/trunk/src/java/org/apache/nutch/webui/client/model/Crawl.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/model/Crawl.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/client/model/Crawl.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/client/model/Crawl.java Mon Sep
28 18:58:33 2015
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.client.model;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+import org.apache.nutch.webui.model.SeedList;
+
+import com.j256.ormlite.field.DatabaseField;
+
+@Entity
+public class Crawl implements Serializable {
+ public enum CrawlStatus {
+ NEW, CRAWLING, FINISHED, ERROR
+ }
+
+ @Id
+ @GeneratedValue
+ private Long id;
+
+ @Column
+ private String crawlId;
+
+ @Column
+ private String crawlName;
+
+ @Column
+ private CrawlStatus status = CrawlStatus.NEW;
+
+ @Column
+ private Integer numberOfRounds = 1;
+
+ @Column
+ @DatabaseField(foreign = true, foreignAutoRefresh = true)
+ private SeedList seedList;
+
+ @Column
+ private String seedDirectory;
+
+ @Column
+ private int progress;
+
+ public Integer getNumberOfRounds() {
+ return numberOfRounds;
+ }
+
+ public void setNumberOfRounds(Integer numberOfRounds) {
+ this.numberOfRounds = numberOfRounds;
+ }
+
+ public String getCrawlId() {
+ return crawlId;
+ }
+
+ public void setCrawlId(String crawlId) {
+ this.crawlId = crawlId;
+ }
+
+ public CrawlStatus getStatus() {
+ return status;
+ }
+
+ public void setStatus(CrawlStatus status) {
+ this.status = status;
+ }
+
+ public String getCrawlName() {
+ return crawlName;
+ }
+
+ public void setCrawlName(String crawlName) {
+ this.crawlName = crawlName;
+ }
+
+ public SeedList getSeedList() {
+ return seedList;
+ }
+
+ public void setSeedList(SeedList seedList) {
+ this.seedList = seedList;
+ }
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getSeedDirectory() {
+ return seedDirectory;
+ }
+
+ public void setSeedDirectory(String seedDirectory) {
+ this.seedDirectory = seedDirectory;
+ }
+
+ public int getProgress() {
+ return progress;
+ }
+
+ public void setProgress(int progress) {
+ this.progress = progress;
+ }
+
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/client/model/JobConfig.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/model/JobConfig.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/client/model/JobConfig.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/client/model/JobConfig.java Mon
Sep 28 18:58:33 2015
@@ -0,0 +1,77 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
******************************************************************************/
+package org.apache.nutch.webui.client.model;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.nutch.webui.client.model.JobInfo.JobType;
+
+import com.google.common.collect.Maps;
+
+public class JobConfig implements Serializable {
+ private String crawlId;
+ private JobType type;
+ private String confId = "default";
+ private String jobClassName;
+ private Map<String, Object> args = Maps.newHashMap();
+
+ public void setArgument(String key, String value) {
+ args.put(key, value);
+ }
+
+ public String getCrawlId() {
+ return crawlId;
+ }
+
+ public void setCrawlId(String crawlId) {
+ this.crawlId = crawlId;
+ }
+
+ public JobType getType() {
+ return type;
+ }
+
+ public void setType(JobType type) {
+ this.type = type;
+ }
+
+ public String getConfId() {
+ return confId;
+ }
+
+ public void setConfId(String confId) {
+ this.confId = confId;
+ }
+
+ public Map<String, Object> getArgs() {
+ return Collections.unmodifiableMap(args);
+ }
+
+ public void setArgs(Map<String, Object> args) {
+ this.args = args;
+ }
+
+ public String getJobClassName() {
+ return jobClassName;
+ }
+
+ public void setJobClassName(String jobClass) {
+ this.jobClassName = jobClass;
+ }
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/client/model/JobInfo.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/model/JobInfo.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/client/model/JobInfo.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/client/model/JobInfo.java Mon
Sep 28 18:58:33 2015
@@ -0,0 +1,104 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
******************************************************************************/
+package org.apache.nutch.webui.client.model;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class JobInfo implements Serializable {
+ public static enum JobType {
+ INJECT, GENERATE, FETCH, PARSE, UPDATEDB, INDEX, READDB, CLASS
+ };
+
+ public static enum State {
+ IDLE, RUNNING, FINISHED, FAILED, KILLED, STOPPING, KILLING, ANY
+ };
+
+ private String id;
+ private String type;
+ private String confId;
+ private Map<String, Object> args;
+ private Map<String, Object> result;
+ private State state;
+ private String msg;
+ private String crawlId;
+
+ public String getMsg() {
+ return msg;
+ }
+
+ public void setMsg(String msg) {
+ this.msg = msg;
+ }
+
+ public State getState() {
+ return state;
+ }
+
+ public void setState(State state) {
+ this.state = state;
+ }
+
+ public Map<String, Object> getResult() {
+ return result;
+ }
+
+ public void setResult(Map<String, Object> result) {
+ this.result = result;
+ }
+
+ public Map<String, Object> getArgs() {
+ return args;
+ }
+
+ public void setArgs(Map<String, Object> args) {
+ this.args = args;
+ }
+
+ public String getConfId() {
+ return confId;
+ }
+
+ public void setConfId(String confId) {
+ this.confId = confId;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getCrawlId() {
+ return crawlId;
+ }
+
+ public void setCrawlId(String crawlId) {
+ this.crawlId = crawlId;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/client/model/NutchStatus.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/client/model/NutchStatus.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/client/model/NutchStatus.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/client/model/NutchStatus.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,62 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
******************************************************************************/
+package org.apache.nutch.webui.client.model;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Set;
+
+public class NutchStatus implements Serializable {
+
+ private Date startDate;
+ private Set<String> configuration;
+ private Collection<JobInfo> jobs;
+ private Collection<JobInfo> runningJobs;
+
+ public Date getStartDate() {
+ return startDate;
+ }
+
+ public void setStartDate(Date startDate) {
+ this.startDate = startDate;
+ }
+
+ public Set<String> getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Set<String> configuration) {
+ this.configuration = configuration;
+ }
+
+ public Collection<JobInfo> getJobs() {
+ return jobs;
+ }
+
+ public void setJobs(Collection<JobInfo> jobs) {
+ this.jobs = jobs;
+ }
+
+ public Collection<JobInfo> getRunningJobs() {
+ return runningJobs;
+ }
+
+ public void setRunningJobs(Collection<JobInfo> runningJobs) {
+ this.runningJobs = runningJobs;
+ }
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/config/CustomDaoFactory.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/config/CustomDaoFactory.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/config/CustomDaoFactory.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/config/CustomDaoFactory.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.config;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.j256.ormlite.dao.Dao;
+import com.j256.ormlite.spring.DaoFactory;
+import com.j256.ormlite.support.ConnectionSource;
+
+public class CustomDaoFactory {
+ private ConnectionSource connectionSource;
+ private List<Dao<?, ?>> registredDaos = Collections
+ .synchronizedList(new ArrayList<Dao<?, ?>>());
+
+ public CustomDaoFactory(ConnectionSource connectionSource) {
+ this.connectionSource = connectionSource;
+ }
+
+ public <T, ID> Dao<T, ID> createDao(Class<T> clazz) {
+ try {
+ Dao<T, ID> dao = DaoFactory.createDao(connectionSource, clazz);
+ register(dao);
+ return dao;
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private <T, ID> void register(Dao<T, ID> dao) {
+ synchronized (registredDaos) {
+ registredDaos.add(dao);
+ }
+ }
+
+ public List<Dao<?, ?>> getCreatedDaos() {
+ synchronized (registredDaos) {
+ return Collections.unmodifiableList(registredDaos);
+ }
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/config/CustomTableCreator.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/config/CustomTableCreator.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/config/CustomTableCreator.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/config/CustomTableCreator.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.config;
+
+import java.sql.SQLException;
+import java.util.List;
+
+import com.j256.ormlite.dao.BaseDaoImpl;
+import com.j256.ormlite.dao.Dao;
+import com.j256.ormlite.support.ConnectionSource;
+import com.j256.ormlite.table.DatabaseTableConfig;
+import com.j256.ormlite.table.TableUtils;
+
+public class CustomTableCreator {
+
+ private ConnectionSource connectionSource;
+ private List<Dao<?, ?>> configuredDaos;
+
+ public CustomTableCreator(ConnectionSource connectionSource,
+ List<Dao<?, ?>> configuredDaos) {
+ this.connectionSource = connectionSource;
+ this.configuredDaos = configuredDaos;
+ initialize();
+ }
+
+ private void initialize() {
+ if (configuredDaos == null) {
+ throw new IllegalStateException("configuredDaos was not set in "
+ + getClass().getSimpleName());
+ }
+
+ for (Dao<?, ?> dao : configuredDaos) {
+ createTableForDao(dao);
+ }
+ }
+
+ private void createTableForDao(Dao<?, ?> dao) {
+ DatabaseTableConfig<?> tableConfig = getTableConfig(dao);
+ createTableIfNotExists(tableConfig);
+ }
+
+ private DatabaseTableConfig<?> getTableConfig(Dao<?, ?> dao) {
+ Class<?> clazz = dao.getDataClass();
+ DatabaseTableConfig<?> tableConfig = null;
+ if (dao instanceof BaseDaoImpl) {
+ tableConfig = ((BaseDaoImpl<?, ?>) dao).getTableConfig();
+ }
+ if (tableConfig == null) {
+ return getConfigFromClass(clazz);
+ }
+ return tableConfig;
+ }
+
+ private DatabaseTableConfig<?> getConfigFromClass(Class<?> clazz) {
+ try {
+ return DatabaseTableConfig.fromClass(connectionSource, clazz);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void createTableIfNotExists(DatabaseTableConfig<?> tableConfig) {
+ try {
+ TableUtils.createTableIfNotExists(connectionSource, tableConfig);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/config/NutchGuiConfiguration.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/config/NutchGuiConfiguration.java?rev=1705744&view=auto
==============================================================================
---
nutch/trunk/src/java/org/apache/nutch/webui/config/NutchGuiConfiguration.java
(added)
+++
nutch/trunk/src/java/org/apache/nutch/webui/config/NutchGuiConfiguration.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.config;
+
+import java.util.List;
+
+import org.apache.nutch.webui.model.NutchInstance;
+
+public class NutchGuiConfiguration {
+ private List<NutchInstance> instances;
+
+ public List<NutchInstance> getInstances() {
+ return instances;
+ }
+
+ public void setInstances(List<NutchInstance> instances) {
+ this.instances = instances;
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/webui/config/SpringConfiguration.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/config/SpringConfiguration.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/config/SpringConfiguration.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/config/SpringConfiguration.java
Mon Sep 28 18:58:33 2015
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.config;
+
+import java.sql.SQLException;
+import java.util.concurrent.Executor;
+
+import org.apache.nutch.webui.client.model.Crawl;
+import org.apache.nutch.webui.model.NutchInstance;
+import org.apache.nutch.webui.model.SeedList;
+import org.apache.nutch.webui.model.SeedUrl;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.AsyncConfigurer;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+import com.j256.ormlite.dao.Dao;
+import com.j256.ormlite.db.H2DatabaseType;
+import com.j256.ormlite.jdbc.JdbcConnectionSource;
+
+@Configuration
+@EnableAsync
+public class SpringConfiguration implements AsyncConfigurer {
+
+ @Override
+ public Executor getAsyncExecutor() {
+ // TODO move magic numbers to properties file
+ ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+ executor.setCorePoolSize(7);
+ executor.setMaxPoolSize(42);
+ executor.setQueueCapacity(11);
+ executor.setThreadNamePrefix("SpringExecutor-");
+ executor.initialize();
+ return executor;
+ }
+
+ @Bean
+ public JdbcConnectionSource getConnectionSource() throws SQLException {
+ JdbcConnectionSource source = new JdbcConnectionSource(
+ "jdbc:h2:~/.nutch/config", new H2DatabaseType());
+ source.initialize();
+ return source;
+ }
+
+ @Bean
+ public CustomDaoFactory getDaoFactory() throws SQLException {
+ return new CustomDaoFactory(getConnectionSource());
+ }
+
+ @Bean
+ public Dao<NutchInstance, Long> createNutchDao() throws SQLException {
+ return getDaoFactory().createDao(NutchInstance.class);
+ }
+
+ @Bean
+ public Dao<SeedList, Long> createSeedListDao() throws SQLException {
+ return getDaoFactory().createDao(SeedList.class);
+ }
+
+ @Bean
+ public Dao<SeedUrl, Long> createSeedUrlDao() throws SQLException {
+ return getDaoFactory().createDao(SeedUrl.class);
+ }
+
+ @Bean
+ public Dao<Crawl, Long> createCrawlDao() throws SQLException {
+ return getDaoFactory().createDao(Crawl.class);
+ }
+
+ @Bean
+ public CustomTableCreator createTableCreator() throws SQLException {
+ return new CustomTableCreator(getConnectionSource(), getDaoFactory()
+ .getCreatedDaos());
+ }
+
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/model/NutchConfig.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/model/NutchConfig.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/model/NutchConfig.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/model/NutchConfig.java Mon Sep
28 18:58:33 2015
@@ -0,0 +1,24 @@
+package org.apache.nutch.webui.model;
+
+import java.io.Serializable;
+
+public class NutchConfig implements Serializable {
+ private String name = "name";
+ private String value;
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return this.name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/model/NutchInstance.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/model/NutchInstance.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/model/NutchInstance.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/model/NutchInstance.java Mon
Sep 28 18:58:33 2015
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.model;
+
+import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+import org.apache.nutch.webui.client.model.ConnectionStatus;
+
+@Entity
+public class NutchInstance implements Serializable {
+
+ @Id
+ @GeneratedValue
+ private Long id;
+
+ @Column
+ private String name = "localhost";
+
+ @Column
+ private String host = "localhost";
+
+ @Column
+ private Integer port = 8081;
+
+ @Column
+ private String username;
+
+ @Column
+ private String password;
+
+ private ConnectionStatus connectionStatus;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public void setHost(String host) {
+ this.host = host;
+ }
+
+ public Integer getPort() {
+ return port;
+ }
+
+ public void setPort(Integer port) {
+ this.port = port;
+ }
+
+ public ConnectionStatus getConnectionStatus() {
+ return connectionStatus;
+ }
+
+ public void setConnectionStatus(ConnectionStatus connectionStatus) {
+ this.connectionStatus = connectionStatus;
+ }
+
+ public URI getUrl() {
+ try {
+ return new URI("http", null, host, port, null, null, null);
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Cannot parse url parameters", e);
+ }
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/model/SeedList.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/model/SeedList.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/model/SeedList.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/model/SeedList.java Mon Sep 28
18:58:33 2015
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.model;
+
+import java.io.Serializable;
+import java.util.Collection;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+import javax.persistence.OneToMany;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.codehaus.jackson.annotate.JsonIgnore;
+
+import com.fasterxml.jackson.annotation.JsonManagedReference;
+import com.j256.ormlite.field.ForeignCollectionField;
+
+@Entity
+public class SeedList implements Serializable {
+
+ @Id
+ @GeneratedValue
+ private Long id;
+
+ @Column
+ private String name;
+
+ @OneToMany
+ @ForeignCollectionField(eager = true)
+ @JsonManagedReference
+ private Collection<SeedUrl> seedUrls;
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ @JsonIgnore
+ public int getSeedUrlsCount() {
+ if (CollectionUtils.isEmpty(seedUrls)) {
+ return 0;
+ }
+ return seedUrls.size();
+ }
+
+ public Collection<SeedUrl> getSeedUrls() {
+ return seedUrls;
+ }
+
+ public void setSeedUrls(Collection<SeedUrl> seedUrls) {
+ this.seedUrls = seedUrls;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((id == null) ? 0 : id.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ SeedList other = (SeedList) obj;
+ if (id == null) {
+ if (other.id != null)
+ return false;
+ } else if (!id.equals(other.id))
+ return false;
+ return true;
+ }
+
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/model/SeedUrl.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/model/SeedUrl.java?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/model/SeedUrl.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/model/SeedUrl.java Mon Sep 28
18:58:33 2015
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.webui.model;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+import org.codehaus.jackson.annotate.JsonIgnore;
+
+import com.fasterxml.jackson.annotation.JsonBackReference;
+import com.j256.ormlite.field.DatabaseField;
+
+@Entity
+public class SeedUrl implements Serializable {
+
+ @Id
+ @GeneratedValue
+ private Long id;
+
+ @Column
+ @DatabaseField(foreign = true, foreignAutoCreate = true, foreignAutoRefresh
= true)
+ @JsonBackReference
+ private SeedList seedList;
+
+ @Column
+ private String url;
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ @JsonIgnore
+ public SeedList getSeedList() {
+ return seedList;
+ }
+
+ @JsonIgnore
+ public void setSeedList(SeedList seedList) {
+ this.seedList = seedList;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((id == null) ? 0 : id.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ SeedUrl other = (SeedUrl) obj;
+ if (id == null) {
+ if (other.id != null)
+ return false;
+ } else if (!id.equals(other.id))
+ return false;
+ return true;
+ }
+}
Added: nutch/trunk/src/java/org/apache/nutch/webui/pages/AbstractBasePage.html
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/pages/AbstractBasePage.html?rev=1705744&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/webui/pages/AbstractBasePage.html
(added)
+++ nutch/trunk/src/java/org/apache/nutch/webui/pages/AbstractBasePage.html Mon
Sep 28 18:58:33 2015
@@ -0,0 +1,33 @@
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor
+ license agreements. See the NOTICE file distributed with this work for
additional
+ information regarding copyright ownership. The ASF licenses this file
to
+ You under the Apache License, Version 2.0 (the "License"); you may not
use
+ this file except in compliance with the License. You may obtain a copy
of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
required
+ by applicable law or agreed to in writing, software distributed under
the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the
specific
+ language governing permissions and limitations under the License. -->
+<!DOCTYPE html>
+<html xmlns:wicket="http://wicket.apache.org";>
+<head>
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta name="description" content="">
+<meta name="author" content="">
+<link rel="shortcut icon" href="../../assets/ico/favicon.ico">
+
+<title>Apache Nutch</title>
+
+</head>
+<body>
+ <div id="wrapper">
+ <nav wicket:id="navigation" class="bs-docs-nav"></nav>
+ <div id="page-wrapper">
+ <div wicket:id="globalNotificationPanel"></div>
+ <wicket:child></wicket:child>
+ </div>
+ </div>
+</body>
+</html>
\ No newline at end of file
