Author: sujen
Date: Sun Oct 25 18:10:03 2015
New Revision: 1710468
URL: http://svn.apache.org/viewvc?rev=1710468&view=rev
Log:
NUTCH-2149 REST endpoint to read Nutch sequence files (Sujen Shah)
Added:
nutch/trunk/src/java/org/apache/nutch/service/NutchReader.java
nutch/trunk/src/java/org/apache/nutch/service/impl/LinkReader.java
nutch/trunk/src/java/org/apache/nutch/service/impl/NodeReader.java
nutch/trunk/src/java/org/apache/nutch/service/impl/SequenceReader.java
nutch/trunk/src/java/org/apache/nutch/service/model/request/ReaderConfig.java
nutch/trunk/src/java/org/apache/nutch/service/resources/ReaderResouce.java
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/service/NutchServer.java
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1710468&r1=1710467&r2=1710468&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Sun Oct 25 18:10:03 2015
@@ -2,6 +2,8 @@ Nutch Change Log
Nutch Current Development 1.11-SNAPSHOT
+* NUTCH-2149 REST endpoint to read Nutch sequence files (Sujen Shah)
+
* NUTCH-2139 Basic plugin to index inlinks and outlinks (jorgelbg)
* NUTCH-2128 Review and update mapred --> mapreduce config params in crawl
script (lewismc)
Added: nutch/trunk/src/java/org/apache/nutch/service/NutchReader.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/NutchReader.java?rev=1710468&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/NutchReader.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/service/NutchReader.java Sun Oct 25
18:10:03 2015
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service;
+
+import java.io.FileNotFoundException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.service.impl.SequenceReader;
+import org.apache.nutch.util.NutchConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public interface NutchReader {
+
+ public static final Logger LOG = LoggerFactory.getLogger(NutchReader.class);
+ public static final Configuration conf = NutchConfiguration.create();
+
+ public List read(String path) throws FileNotFoundException;
+ public List head(String path, int nrows) throws FileNotFoundException;
+ public List slice(String path, int start, int end) throws
FileNotFoundException;
+ public int count(String path) throws FileNotFoundException;
+}
Modified: nutch/trunk/src/java/org/apache/nutch/service/NutchServer.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/NutchServer.java?rev=1710468&r1=1710467&r2=1710468&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/NutchServer.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/service/NutchServer.java Sun Oct 25
18:10:03 2015
@@ -49,6 +49,7 @@ import org.apache.nutch.service.resource
import org.apache.nutch.service.resources.ConfigResource;
import org.apache.nutch.service.resources.DbResource;
import org.apache.nutch.service.resources.JobResource;
+import org.apache.nutch.service.resources.ReaderResouce;
import org.apache.nutch.service.resources.SeedResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -134,6 +135,7 @@ public class NutchServer {
resources.add(DbResource.class);
resources.add(AdminResource.class);
resources.add(SeedResource.class);
+ resources.add(ReaderResouce.class);
return resources;
}
Added: nutch/trunk/src/java/org/apache/nutch/service/impl/LinkReader.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/impl/LinkReader.java?rev=1710468&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/impl/LinkReader.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/service/impl/LinkReader.java Sun Oct
25 18:10:03 2015
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.impl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import javax.ws.rs.WebApplicationException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.SequenceFile.Reader;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.nutch.scoring.webgraph.LinkDatum;
+import org.apache.nutch.service.NutchReader;
+
+public class LinkReader implements NutchReader{
+
+ @Override
+ public List read(String path) throws FileNotFoundException {
+ List<HashMap> rows=new ArrayList<HashMap>();
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ try{
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key = (Writable)
+ ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ LinkDatum value = new LinkDatum();
+
+ while(reader.next(key, value)) {
+ try {
+ HashMap<String, String> t_row = getLinksRow(key,value);
+ rows.add(t_row);
+ }
+ catch (Exception e) {
+ }
+ }
+ reader.close();
+
+ }catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ LOG.error("Error occurred while reading file {} : ", file,
StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+
+ return rows;
+ }
+
+ @Override
+ public List head(String path, int nrows) throws FileNotFoundException {
+ List<HashMap> rows=new ArrayList<HashMap>();
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ try{
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key = (Writable)
+ ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ LinkDatum value = new LinkDatum();
+ int i = 0;
+ while(reader.next(key, value) && i<nrows) {
+
+ HashMap<String, String> t_row = getLinksRow(key,value);
+ rows.add(t_row);
+
+ i++;
+ }
+ reader.close();
+
+ }catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ LOG.error("Error occurred while reading file {} : ", file,
StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+
+ return rows;
+ }
+
+ @Override
+ public List slice(String path, int start, int end)
+ throws FileNotFoundException {
+ List<HashMap> rows=new ArrayList<HashMap>();
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ try{
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key = (Writable)
+ ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ LinkDatum value = new LinkDatum();
+ int i = 0;
+
+ for(;i<start && reader.next(key, value);i++){} // increment to read
start position
+ while(reader.next(key, value) && i<end) {
+ HashMap<String, String> t_row = getLinksRow(key,value);
+ rows.add(t_row);
+
+ i++;
+ }
+ reader.close();
+
+ }catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ LOG.error("Error occurred while reading file {} : ", file,
StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+
+ return rows;
+ }
+
+ @Override
+ public int count(String path) throws FileNotFoundException {
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ int i = 0;
+ try {
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key =
(Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ Writable value =
(Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
+
+ while(reader.next(key, value)) {
+ i++;
+ }
+ reader.close();
+ } catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ LOG.error("Error occurred while reading file {} : ", file,
StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+ return i;
+ }
+
+ private HashMap<String, String> getLinksRow(Writable key, LinkDatum value) {
+ HashMap<String, String> t_row = new HashMap<String, String>();
+ t_row.put("key_url", key.toString());
+ t_row.put("url", value.getUrl());
+ t_row.put("anchor", value.getAnchor());
+ t_row.put("score", String.valueOf(value.getScore()));
+ t_row.put("timestamp", String.valueOf(value.getTimestamp()));
+ t_row.put("linktype", String.valueOf(value.getLinkType()));
+
+ return t_row;
+ }
+}
Added: nutch/trunk/src/java/org/apache/nutch/service/impl/NodeReader.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/impl/NodeReader.java?rev=1710468&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/impl/NodeReader.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/service/impl/NodeReader.java Sun Oct
25 18:10:03 2015
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.impl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import javax.ws.rs.WebApplicationException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.io.SequenceFile.Reader;
+import org.apache.nutch.scoring.webgraph.Node;
+import org.apache.nutch.service.NutchReader;
+
+public class NodeReader implements NutchReader {
+
+ @Override
+ public List read(String path) throws FileNotFoundException {
+ // TODO Auto-generated method stub
+ List<HashMap> rows=new ArrayList<HashMap>();
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ try{
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key = (Writable)
+ ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ Node value = new Node();
+
+ while(reader.next(key, value)) {
+ try {
+ HashMap<String, String> t_row = getNodeRow(key,value);
+ rows.add(t_row);
+ }
+ catch (Exception e) {
+ }
+ }
+ reader.close();
+
+ }catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ LOG.error("Error occurred while reading file {} : ", file,
StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+
+ return rows;
+
+ }
+
+ @Override
+ public List head(String path, int nrows) throws FileNotFoundException {
+ List<HashMap> rows=new ArrayList<HashMap>();
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ try{
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key = (Writable)
+ ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ Node value = new Node();
+ int i = 0;
+ while(reader.next(key, value) && i<nrows) {
+ HashMap<String, String> t_row = getNodeRow(key,value);
+ rows.add(t_row);
+
+ i++;
+ }
+ reader.close();
+
+ }catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ LOG.error("Error occurred while reading file {} : ", file,
+ StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+
+ return rows;
+ }
+
+ @Override
+ public List slice(String path, int start, int end)
+ throws FileNotFoundException {
+ List<HashMap> rows=new ArrayList<HashMap>();
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ try{
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key = (Writable)
+ ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ Node value = new Node();
+ int i = 0;
+
+ for(;i<start && reader.next(key, value);i++){} // increment to read
start position
+ while(reader.next(key, value) && i<end) {
+ HashMap<String, String> t_row = getNodeRow(key,value);
+ rows.add(t_row);
+
+ i++;
+ }
+ reader.close();
+
+ }catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ LOG.error("Error occurred while reading file {} : ", file,
+ StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+
+ return rows;
+ }
+
+ @Override
+ public int count(String path) throws FileNotFoundException {
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ int i =0;
+ try{
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key = (Writable)
+ ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ Node value = new Node();
+
+ while(reader.next(key, value)) {
+ i++;
+ }
+ reader.close();
+
+ }catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ LOG.error("Error occurred while reading file {} : ", file,
+ StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+
+ return i;
+ }
+
+ private HashMap<String, String> getNodeRow(Writable key, Node value) {
+ HashMap<String, String> t_row = new HashMap<String, String>();
+ t_row.put("key_url", key.toString());
+ t_row.put("num_inlinks", String.valueOf(value.getNumInlinks()) );
+ t_row.put("num_outlinks", String.valueOf(value.getNumOutlinks()) );
+ t_row.put("inlink_score", String.valueOf(value.getInlinkScore()));
+ t_row.put("outlink_score", String.valueOf(value.getOutlinkScore()));
+ t_row.put("metadata", value.getMetadata().toString());
+
+ return t_row;
+ }
+}
Added: nutch/trunk/src/java/org/apache/nutch/service/impl/SequenceReader.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/impl/SequenceReader.java?rev=1710468&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/impl/SequenceReader.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/service/impl/SequenceReader.java Sun
Oct 25 18:10:03 2015
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.impl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.ws.rs.WebApplicationException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.Reader;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.nutch.service.NutchReader;
+
+/**
+ * Enables reading a sequence file and methods provide different
+ * ways to read the file.
+ * @author Sujen Shah
+ *
+ */
+public class SequenceReader implements NutchReader {
+
+ @Override
+ public List<List<String>> read(String path) throws FileNotFoundException {
+ // TODO Auto-generated method stub
+ List<List<String>> rows=new ArrayList<List<String>>();
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ try {
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key =
+ (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ Writable value =
+ (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
+
+ while(reader.next(key, value)) {
+ List<String> row =new ArrayList<String>();
+ row.add(key.toString());
+ row.add(value.toString());
+ rows.add(row);
+ }
+ reader.close();
+ }catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ LOG.error("Error occurred while reading file {} : ", file,
+ StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+ return rows;
+ }
+
+ @Override
+ public List<List<String>> head(String path, int nrows)
+ throws FileNotFoundException {
+ // TODO Auto-generated method stub
+
+ List<List<String>> rows=new ArrayList<List<String>>();
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ try {
+
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key =
+ (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ Writable value =
+ (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
+ int i = 0;
+ while(reader.next(key, value) && i<nrows) {
+ List<String> row =new ArrayList<String>();
+ row.add(key.toString());
+ row.add(value.toString());
+ rows.add(row);
+ i++;
+ }
+ reader.close();
+ } catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ LOG.error("Error occurred while reading file {} : ", file,
+ StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+ return rows;
+ }
+
+ @Override
+ public List<List<String>> slice(String path, int start, int end)
+ throws FileNotFoundException {
+ List<List<String>> rows=new ArrayList<List<String>>();
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ try {
+
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key =
+ (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ Writable value =
+ (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
+ int i = 0;
+
+ for(;i<start && reader.next(key, value);i++){} // increment to read
start position
+ while(reader.next(key, value) && i<end) {
+ List<String> row =new ArrayList<String>();
+ row.add(key.toString());
+ row.add(value.toString());
+ rows.add(row);
+ i++;
+ }
+ reader.close();
+ } catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ LOG.error("Error occurred while reading file {} : ", file,
+ StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+ return rows;
+ }
+
+ @Override
+ public int count(String path) throws FileNotFoundException {
+ Path file = new Path(path);
+ SequenceFile.Reader reader;
+ int i = 0;
+ try {
+ reader = new SequenceFile.Reader(conf, Reader.file(file));
+ Writable key =
+ (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ Writable value =
+ (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
+
+ while(reader.next(key, value)) {
+ i++;
+ }
+ reader.close();
+ } catch(FileNotFoundException fne){
+ throw new FileNotFoundException();
+ }catch (IOException e) {
+ // TODO Auto-generated catch block
+ LOG.error("Error occurred while reading file {} : ", file,
+ StringUtils.stringifyException(e));
+ throw new WebApplicationException();
+ }
+ return i;
+ }
+
+}
Added:
nutch/trunk/src/java/org/apache/nutch/service/model/request/ReaderConfig.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/model/request/ReaderConfig.java?rev=1710468&view=auto
==============================================================================
---
nutch/trunk/src/java/org/apache/nutch/service/model/request/ReaderConfig.java
(added)
+++
nutch/trunk/src/java/org/apache/nutch/service/model/request/ReaderConfig.java
Sun Oct 25 18:10:03 2015
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.model.request;
+
+public class ReaderConfig {
+
+ private String path;
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+}
Added:
nutch/trunk/src/java/org/apache/nutch/service/resources/ReaderResouce.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/service/resources/ReaderResouce.java?rev=1710468&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/service/resources/ReaderResouce.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/service/resources/ReaderResouce.java
Sun Oct 25 18:10:03 2015
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.service.resources;
+
+import java.util.HashMap;
+
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import org.apache.nutch.service.NutchReader;
+import org.apache.nutch.service.impl.LinkReader;
+import org.apache.nutch.service.impl.NodeReader;
+import org.apache.nutch.service.impl.SequenceReader;
+import org.apache.nutch.service.model.request.ReaderConfig;
+
+/**
+ * The Reader endpoint enables a user to read sequence files,
+ * nodes and links from the Nutch webgraph.
+ * @author Sujen Shah
+ *
+ */
+@Path("/reader")
+public class ReaderResouce {
+
+ @Path("/sequence/read")
+ @POST
+ public Response seqRead(ReaderConfig readerConf,
+ @DefaultValue("-1")@QueryParam("nrows") int nrows,
+ @DefaultValue("-1")@QueryParam("start") int start,
+ @QueryParam("end")int end, @QueryParam("count") boolean count) {
+
+ NutchReader reader = new SequenceReader();
+ String path = readerConf.getPath();
+ return performRead(reader, path, nrows, start, end, count);
+ }
+
+ @Path("/link")
+ @GET
+ public Response linkRead() {
+ HashMap<String, String> schema = new HashMap<>();
+ schema.put("key_url","string");
+ schema.put("timestamp", "int");
+ schema.put("score","float");
+ schema.put("anchor","string");
+ schema.put("linktype","string");
+ schema.put("url","string");
+ return Response.ok(schema).type(MediaType.APPLICATION_JSON).build();
+ }
+
+ @Path("/link/read")
+ @POST
+ public Response linkRead(ReaderConfig readerConf,
+ @DefaultValue("-1")@QueryParam("nrows") int nrows,
+ @DefaultValue("-1")@QueryParam("start") int start,
+ @QueryParam("end") int end, @QueryParam("count") boolean count) {
+
+ NutchReader reader = new LinkReader();
+ String path = readerConf.getPath();
+ return performRead(reader, path, nrows, start, end, count);
+ }
+
+ @Path("/node")
+ @GET
+ public Response nodeRead() {
+ HashMap<String, String> schema = new HashMap<>();
+ schema.put("key_url","string");
+ schema.put("num_inlinks", "int");
+ schema.put("num_outlinks","int");
+ schema.put("inlink_score","float");
+ schema.put("outlink_score","float");
+ schema.put("metadata","string");
+ return Response.ok(schema).type(MediaType.APPLICATION_JSON).build();
+ }
+
+
+ @Path("/node/read")
+ @POST
+ public Response nodeRead(ReaderConfig readerConf,
+ @DefaultValue("-1")@QueryParam("nrows") int nrows,
+ @DefaultValue("-1")@QueryParam("start") int start,
+ @QueryParam("end") int end, @QueryParam("count") boolean count) {
+
+ NutchReader reader = new NodeReader();
+ String path = readerConf.getPath();
+ return performRead(reader, path, nrows, start, end, count);
+ }
+
+
+ private Response performRead(NutchReader reader, String path,
+ int nrows, int start, int end, boolean count) {
+ Object result;
+ try{
+ if(count){
+ result = reader.count(path);
+ return Response.ok(result).type(MediaType.TEXT_PLAIN).build();
+ }
+ else if(start>-1 && end>0) {
+ result = reader.slice(path, start, end);
+ }
+ else if(nrows>-1) {
+ result = reader.head(path, nrows);
+ }
+ else {
+ result = reader.read(path);
+ }
+ return Response.ok(result).type(MediaType.APPLICATION_JSON).build();
+ }catch(Exception e){
+ return Response.status(Status.BAD_REQUEST).entity("File not
found").build();
+ }
+ }
+
+}
