Author: mattmann
Date: Thu Mar 19 03:46:16 2015
New Revision: 1667649
URL: http://svn.apache.org/r1667649
Log:
Fix for NUTCH-1966 Configuration endpoint for 1x REST API contributed by Sujen
Shah <[email protected]> this closes #13.
Added:
nutch/trunk/src/java/org/apache/nutch/service/
nutch/trunk/src/java/org/apache/nutch/service/ConfManager.java
nutch/trunk/src/java/org/apache/nutch/service/JobManager.java
nutch/trunk/src/java/org/apache/nutch/service/NutchServer.java
nutch/trunk/src/java/org/apache/nutch/service/impl/
nutch/trunk/src/java/org/apache/nutch/service/impl/ConfManagerImpl.java
nutch/trunk/src/java/org/apache/nutch/service/model/
nutch/trunk/src/java/org/apache/nutch/service/model/request/
nutch/trunk/src/java/org/apache/nutch/service/model/request/NutchConfig.java
nutch/trunk/src/java/org/apache/nutch/service/model/response/
nutch/trunk/src/java/org/apache/nutch/service/model/response/JobConfig.java
nutch/trunk/src/java/org/apache/nutch/service/model/response/JobInfo.java
nutch/trunk/src/java/org/apache/nutch/service/resources/
nutch/trunk/src/java/org/apache/nutch/service/resources/AbstractResource.java
nutch/trunk/src/java/org/apache/nutch/service/resources/ConfigResource.java
nutch/trunk/src/java/org/apache/nutch/service/resources/JobResource.java
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/ivy/ivy.xml
nutch/trunk/src/bin/nutch
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1667649&r1=1667648&r2=1667649&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Thu Mar 19 03:46:16 2015
@@ -2,6 +2,8 @@ Nutch Change Log
Nutch Current Development 1.10-SNAPSHOT
+* NUTCH-1966 Configuration endpoint for 1x REST API (Sujen Shah via mattmann)
+
* NUTCH-1967 Possible SIooBE in MimeAdaptiveFetchSchedule (markus)
* NUTCH-1957 FileDumper output file name collisions (Renxia Wang via mattmann)
Modified: nutch/trunk/ivy/ivy.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/ivy/ivy.xml?rev=1667649&r1=1667648&r2=1667649&view=diff
==============================================================================
--- nutch/trunk/ivy/ivy.xml (original)
+++ nutch/trunk/ivy/ivy.xml Thu Mar 19 03:46:16 2015
@@ -71,10 +71,14 @@
<dependency org="com.google.guava" name="guava" rev="11.0.2" />
<dependency org="com.google.code.crawler-commons"
name="crawler-commons"
rev="0.5" />
-
+ <dependency org="org.apache.cxf" name="cxf" rev="3.0.4"/>
+ <dependency org="org.apache.cxf" name="cxf-rt-frontend-jaxws"
rev="3.0.4"/>
+ <dependency org="org.apache.cxf" name="cxf-rt-frontend-jaxrs"
rev="3.0.4"/>
+ <dependency org="org.apache.cxf" name="cxf-rt-transports-http"
rev="3.0.4"/>
+ <dependency org="org.apache.cxf" name="cxf-rt-transports-http-jetty"
rev="3.0.4"/>
<dependency org="com.fasterxml.jackson.core"
name="jackson-databind" rev="2.5.1" />
<dependency org="com.fasterxml.jackson.dataformat"
name="jackson-dataformat-cbor" rev="2.5.1" />
-
+
<!--Configuration: test -->
<!--artifacts needed for testing -->
Modified: nutch/trunk/src/bin/nutch
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/bin/nutch?rev=1667649&r1=1667648&r2=1667649&view=diff
==============================================================================
--- nutch/trunk/src/bin/nutch (original)
+++ nutch/trunk/src/bin/nutch Thu Mar 19 03:46:16 2015
@@ -86,6 +86,7 @@ if [ $# = 0 ]; then
echo " nodedumper dumps the web graph's node scores"
echo " plugin load a plugin and run one of its classes main()"
echo " junit runs the given JUnit test"
+ echo " startserver runs the Nutch Server on localhost:8081"
echo " or"
echo " CLASSNAME run the class named CLASSNAME"
echo "Most commands print help when invoked w/o parameters."
@@ -270,6 +271,8 @@ elif [ "$COMMAND" = "plugin" ] ; then
elif [ "$COMMAND" = "junit" ] ; then
CLASSPATH="$CLASSPATH:$NUTCH_HOME/test/classes/"
CLASS=org.junit.runner.JUnitCore
+elif [ "$COMMAND" = "startserver" ] ; then
+ CLASS=org.apache.nutch.service.NutchServer
else
CLASS=$COMMAND
fi
Added: nutch/trunk/src/java/org/apache/nutch/service/ConfManager.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/ConfManager.java?rev=1667649&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/ConfManager.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/service/ConfManager.java Thu Mar 19
03:46:16 2015
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.service;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.service.model.request.NutchConfig;
+
+public interface ConfManager {
+
+ public Configuration get(String confId);
+
+ public Map<String, String> getAsMap(String confId);
+
+ public void setProperty(String confId, String propName, String propValue);
+
+ public Set<String> list();
+
+ public String create(NutchConfig nutchConfig);
+
+ public void delete(String confId);
+}
Added: nutch/trunk/src/java/org/apache/nutch/service/JobManager.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/JobManager.java?rev=1667649&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/JobManager.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/service/JobManager.java Thu Mar 19
03:46:16 2015
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.service;
+
+import java.util.Collection;
+
+import org.apache.nutch.service.model.response.JobConfig;
+import org.apache.nutch.service.model.response.JobInfo;
+import org.apache.nutch.service.model.response.JobInfo.State;
+
+public interface JobManager {
+
+ public static enum JobType{
+ INJECT, GENERATE, FETCH, PARSE, UPDATEDB, INDEX, READDB, CLASS
+ };
+ public Collection<JobInfo> list(String crawlId, State state);
+
+ public JobInfo get(String crawlId, String id);
+
+ public String create(JobConfig jobConfig);
+
+ public boolean abort(String crawlId, String id);
+
+ public boolean stop(String crawlId, String id);
+}
Added: nutch/trunk/src/java/org/apache/nutch/service/NutchServer.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/NutchServer.java?rev=1667649&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/NutchServer.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/service/NutchServer.java Thu Mar 19
03:46:16 2015
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.service;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.cxf.binding.BindingFactoryManager;
+import org.apache.cxf.jaxrs.JAXRSBindingFactory;
+import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
+import org.apache.cxf.jaxrs.lifecycle.ResourceProvider;
+import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
+import org.apache.nutch.service.impl.ConfManagerImpl;
+import org.apache.nutch.service.resources.ConfigResource;
+import org.apache.nutch.service.resources.JobResource;
+import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NutchServer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(NutchServer.class);
+
+ private static final String LOCALHOST = "localhost";
+ private static final Integer DEFAULT_PORT = 8081;
+ private static final int JOB_CAPACITY = 100;
+
+ private static Integer port = DEFAULT_PORT;
+
+ private static final String CMD_HELP = "help";
+ private static final String CMD_PORT = "port";
+
+ private long started;
+ private boolean running;
+ private ConfManager configManager;
+ private JAXRSServerFactoryBean sf;
+
+ private static NutchServer server;
+
+ static {
+ server = new NutchServer();
+ }
+
+ private NutchServer() {
+ configManager = new ConfManagerImpl();
+
+ sf = new JAXRSServerFactoryBean();
+ BindingFactoryManager manager =
sf.getBus().getExtension(BindingFactoryManager.class);
+ JAXRSBindingFactory factory = new JAXRSBindingFactory();
+ factory.setBus(sf.getBus());
+
manager.registerBindingFactory(JAXRSBindingFactory.JAXRS_BINDING_ID, factory);
+ sf.setResourceClasses(getClasses());
+ sf.setResourceProviders(getResourceProviders());
+ sf.setProvider(new JacksonJaxbJsonProvider());
+
+
+ }
+
+ public static NutchServer getInstance() {
+ return server;
+ }
+
+ private static void startServer() {
+ server.start();
+ }
+
+ private void start() {
+ LOG.info("Starting NutchServer on port: {} ...",port);
+ try{
+ String address = "http://"; + LOCALHOST + ":" + port;
+ sf.setAddress(address);
+ sf.create();
+ }catch(Exception e){
+ throw new IllegalStateException("Server could not be
started", e);
+ }
+
+ started = System.currentTimeMillis();
+ running = true;
+ LOG.info("Started Nutch Server on port {} at {}", port,
started);
+ System.out.println("Started Nutch Server on port " + port + "
at " + started);
+ }
+
+ public List<Class<?>> getClasses() {
+ List<Class<?>> resources = new ArrayList<Class<?>>();
+ resources.add(JobResource.class);
+ resources.add(ConfigResource.class);
+ return resources;
+ }
+
+ public List<ResourceProvider> getResourceProviders() {
+ List<ResourceProvider> resourceProviders = new
ArrayList<ResourceProvider>();
+ resourceProviders.add(new
SingletonResourceProvider(getConfManager()));
+
+ return resourceProviders;
+ }
+
+ public ConfManager getConfManager() {
+ return configManager;
+ }
+
+ public static void main(String[] args) throws ParseException {
+ CommandLineParser parser = new PosixParser();
+ Options options = createOptions();
+ CommandLine commandLine = parser.parse(options, args);
+ if (commandLine.hasOption(CMD_HELP)) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("NutchServer", options, true);
+ return;
+ }
+
+ if (commandLine.hasOption(CMD_PORT)) {
+ port =
Integer.parseInt(commandLine.getOptionValue(CMD_PORT));
+ }
+ startServer();
+ }
+
+ private static Options createOptions() {
+ Options options = new Options();
+
+ OptionBuilder.withDescription("Show this help");
+ options.addOption(OptionBuilder.create(CMD_HELP));
+
+ OptionBuilder.withArgName("port");
+ OptionBuilder.hasOptionalArg();
+ OptionBuilder.withDescription("The port to run the Nutch
Server. Default port 8081");
+ options.addOption(OptionBuilder.create(CMD_PORT));
+ return options;
+ }
+
+}
Added: nutch/trunk/src/java/org/apache/nutch/service/impl/ConfManagerImpl.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/impl/ConfManagerImpl.java?rev=1667649&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/impl/ConfManagerImpl.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/service/impl/ConfManagerImpl.java Thu
Mar 19 03:46:16 2015
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.impl;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.service.ConfManager;
+import org.apache.nutch.service.model.request.NutchConfig;
+import org.apache.nutch.service.resources.ConfigResource;
+import org.apache.nutch.util.NutchConfiguration;
+
+import com.google.common.collect.Maps;
+
+public class ConfManagerImpl implements ConfManager {
+
+
+ private Map<String, Configuration> configurations =
Maps.newConcurrentMap();
+
+ private AtomicInteger newConfigId = new AtomicInteger();
+
+ public ConfManagerImpl() {
+ configurations.put(ConfigResource.DEFAULT,
NutchConfiguration.create());
+ }
+
+ /**
+ * Returns the configuration associatedConfManagerImpl with the given
confId
+ */
+ public Configuration get(String confId) {
+ if (confId == null) {
+ return configurations.get(ConfigResource.DEFAULT);
+ }
+ return configurations.get(confId);
+ }
+
+ public Map<String, String> getAsMap(String confId) {
+ Configuration configuration = configurations.get(confId);
+ if (configuration == null) {
+ return Collections.emptyMap();
+ }
+
+ Iterator<Entry<String, String>> iterator = configuration.iterator();
+ Map<String, String> configMap = Maps.newTreeMap();
+ while (iterator.hasNext()) {
+ Entry<String, String> entry = iterator.next();
+ configMap.put(entry.getKey(), entry.getValue());
+ }
+ return configMap;
+ }
+
+ /**
+ * Sets the given property in the configuration associated with the
confId
+ */
+ public void setProperty(String confId, String propName, String
propValue) {
+ if (!configurations.containsKey(confId)) {
+ throw new IllegalArgumentException("Unknown configId '" + confId
+ "'");
+ }
+ Configuration conf = configurations.get(confId);
+ conf.set(propName, propValue);
+ }
+
+ public Set<String> list() {
+ return configurations.keySet();
+ }
+
+ /**
+ * Created a new configuration based on the values provided.
+ * @param NutchConfig
+ * @return String - confId
+ */
+ public String create(NutchConfig nutchConfig) {
+ if (StringUtils.isBlank(nutchConfig.getConfigId())) {
+
nutchConfig.setConfigId(String.valueOf(newConfigId.incrementAndGet()));
+ }
+
+ if (!canCreate(nutchConfig)) {
+ throw new IllegalArgumentException("Config already exists.");
+ }
+
+ createHadoopConfig(nutchConfig);
+ return nutchConfig.getConfigId();
+ }
+
+
+ public void delete(String confId) {
+ configurations.remove(confId);
+ }
+
+ private boolean canCreate(NutchConfig nutchConfig) {
+ if (nutchConfig.isForce()) {
+ return true;
+ }
+ if (!configurations.containsKey(nutchConfig.getConfigId())) {
+ return true;
+ }
+ return false;
+ }
+
+ private void createHadoopConfig(NutchConfig nutchConfig) {
+ Configuration conf = NutchConfiguration.create();
+ configurations.put(nutchConfig.getConfigId(), conf);
+
+ if (MapUtils.isEmpty(nutchConfig.getParams())) {
+ return;
+ }
+ for (Entry<String, String> e : nutchConfig.getParams().entrySet()) {
+ conf.set(e.getKey(), e.getValue());
+ }
+ }
+
+}
Added:
nutch/trunk/src/java/org/apache/nutch/service/model/request/NutchConfig.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/model/request/NutchConfig.java?rev=1667649&view=auto
==============================================================================
---
nutch/trunk/src/java/org/apache/nutch/service/model/request/NutchConfig.java
(added)
+++
nutch/trunk/src/java/org/apache/nutch/service/model/request/NutchConfig.java
Thu Mar 19 03:46:16 2015
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.model.request;
+
+import java.util.Map;
+
+import java.util.Collections;
+
+public class NutchConfig {
+ private String configId;
+ private boolean force = false;
+ private Map<String, String> params = Collections.emptyMap();
+
+ public Map<String, String> getParams() {
+ return params;
+ }
+
+ public void setParams(Map<String, String> params) {
+ this.params = params;
+ }
+
+ public String getConfigId() {
+ return configId;
+ }
+
+ public void setConfigId(String configId) {
+ this.configId = configId;
+ }
+
+ public boolean isForce() {
+ return force;
+ }
+
+ public void setForce(boolean force) {
+ this.force = force;
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/service/model/response/JobConfig.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/model/response/JobConfig.java?rev=1667649&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/model/response/JobConfig.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/service/model/response/JobConfig.java
Thu Mar 19 03:46:16 2015
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.model.response;
+
+public class JobConfig {
+
+}
Added: nutch/trunk/src/java/org/apache/nutch/service/model/response/JobInfo.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/model/response/JobInfo.java?rev=1667649&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/model/response/JobInfo.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/service/model/response/JobInfo.java
Thu Mar 19 03:46:16 2015
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.model.response;
+
+import java.util.Map;
+
+import org.apache.nutch.service.JobManager.JobType;
+
+/**
+ * This is the response object containing Job information
+ *
+ *
+ */
+public class JobInfo {
+
+ public static enum State {
+ IDLE, RUNNING, FINISHED, FAILED, KILLED, STOPPING, KILLING, ANY
+ };
+
+ private String id;
+ private JobType type;
+ private String confId;
+ private Map<String, Object> args;
+ private Map<String, Object> result;
+ private State state;
+ private String msg;
+ private String crawlId;
+
+ public String getId() {
+ return id;
+ }
+ public void setId(String id) {
+ this.id = id;
+ }
+ public JobType getType() {
+ return type;
+ }
+ public void setType(JobType type) {
+ this.type = type;
+ }
+ public String getConfId() {
+ return confId;
+ }
+ public void setConfId(String confId) {
+ this.confId = confId;
+ }
+ public Map<String, Object> getArgs() {
+ return args;
+ }
+ public void setArgs(Map<String, Object> args) {
+ this.args = args;
+ }
+ public Map<String, Object> getResult() {
+ return result;
+ }
+ public void setResult(Map<String, Object> result) {
+ this.result = result;
+ }
+ public State getState() {
+ return state;
+ }
+ public void setState(State state) {
+ this.state = state;
+ }
+ public String getMsg() {
+ return msg;
+ }
+ public void setMsg(String msg) {
+ this.msg = msg;
+ }
+ public String getCrawlId() {
+ return crawlId;
+ }
+ public void setCrawlId(String crawlId) {
+ this.crawlId = crawlId;
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/service/resources/AbstractResource.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/resources/AbstractResource.java?rev=1667649&view=auto
==============================================================================
---
nutch/trunk/src/java/org/apache/nutch/service/resources/AbstractResource.java
(added)
+++
nutch/trunk/src/java/org/apache/nutch/service/resources/AbstractResource.java
Thu Mar 19 03:46:16 2015
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.resources;
+
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import org.apache.nutch.service.ConfManager;
+import org.apache.nutch.service.JobManager;
+import org.apache.nutch.service.NutchServer;
+
+@Produces(MediaType.APPLICATION_JSON)
+public abstract class AbstractResource {
+
+ protected JobManager jobManager;
+ protected ConfManager configManager;
+
+ public AbstractResource() {
+ configManager = NutchServer.getInstance().getConfManager();
+ }
+
+ protected void throwBadRequestException(String message) {
+ throw new
WebApplicationException(Response.status(Status.BAD_REQUEST).entity(message).build());
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/service/resources/ConfigResource.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/resources/ConfigResource.java?rev=1667649&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/resources/ConfigResource.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/service/resources/ConfigResource.java
Thu Mar 19 03:46:16 2015
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.service.resources;
+
+
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.GenericEntity;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import org.apache.nutch.service.model.request.NutchConfig;
+import org.codehaus.jettison.json.JSONObject;
+
+@Path("/config")
+public class ConfigResource extends AbstractResource{
+
+ public static final String DEFAULT = "default";
+
+ @GET
+ @Path("/")
+ public Set<String> getConfigs() {
+ return configManager.list();
+ }
+
+ @GET
+ @Path("/{configId}")
+ public Map<String, String> getConfig(@PathParam("configId") String
configId) {
+ return configManager.getAsMap(configId);
+ }
+
+ @GET
+ @Path("/{configId}/{propertyId}")
+ public String getProperty(@PathParam("configId") String configId,
+ @PathParam("propertyId") String propertyId) {
+ return configManager.getAsMap(configId).get(propertyId);
+ }
+
+ @DELETE
+ @Path("/{configId}")
+ public void deleteConfig(@PathParam("configId") String configId) {
+ configManager.delete(configId);
+ }
+
+ @POST
+ @Path("/{configId}")
+ @Consumes(MediaType.APPLICATION_JSON)
+ public String createConfig(NutchConfig newConfig) {
+ if (newConfig == null) {
+ throw new
WebApplicationException(Response.status(Status.BAD_REQUEST)
+ .entity("Nutch configuration cannot be
empty!").build());
+ }
+ return configManager.create(newConfig);
+ }
+}
Added: nutch/trunk/src/java/org/apache/nutch/service/resources/JobResource.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/resources/JobResource.java?rev=1667649&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/resources/JobResource.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/service/resources/JobResource.java
Thu Mar 19 03:46:16 2015
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.resources;
+
+import java.util.Collection;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+import org.apache.nutch.service.model.response.JobConfig;
+import org.apache.nutch.service.model.response.JobInfo;
+import org.apache.nutch.service.model.response.JobInfo.State;
+
+@Path(value = "/job")
+public class JobResource extends AbstractResource {
+
+ @GET
+ @Path(value = "/")
+ public Collection<JobInfo> getJobs(@QueryParam("crawlId") String crawlId) {
+ return jobManager.list(crawlId, State.ANY);
+ }
+
+ @GET
+ @Path(value = "/{id}")
+ public JobInfo getInfo(@PathParam("id") String id,
+ @QueryParam("crawlId") String crawlId) {
+ return jobManager.get(crawlId, id);
+ }
+
+ @GET
+ @Path(value = "/{id}/stop")
+ public boolean stop(@PathParam("id") String id,
+ @QueryParam("crawlId") String crawlId) {
+ return jobManager.stop(crawlId, id);
+ }
+
+ @GET
+ @Path(value = "/{id}/abort")
+ public boolean abort(@PathParam("id") String id,
+ @QueryParam("crawlId") String crawlId) {
+ return jobManager.abort(crawlId, id);
+ }
+
+ @POST
+ @Path(value = "/create")
+ @Consumes(MediaType.APPLICATION_JSON)
+ public String create(JobConfig config) {
+ if (config == null) {
+ throwBadRequestException("Job configuration is required!");
+ }
+ return jobManager.create(config);
+ }
+}
