Author: fenglu
Date: Sun Mar 24 14:08:05 2013
New Revision: 1460380
URL: http://svn.apache.org/r1460380
Log:
NUTCH-1533 - Implement getPrevModifiedTime(), setPrevModifiedTime(),
getBatchId() and setBatchId() accessors in o.a.n.storage.WebPage
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/conf/gora-accumulo-mapping.xml
nutch/branches/2.x/conf/gora-cassandra-mapping.xml
nutch/branches/2.x/conf/gora-hbase-mapping.xml
nutch/branches/2.x/conf/gora-sql-mapping.xml
nutch/branches/2.x/src/gora/webpage.avsc
nutch/branches/2.x/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdateReducer.java
nutch/branches/2.x/src/java/org/apache/nutch/crawl/DefaultFetchSchedule.java
nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorReducer.java
nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java
nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java
nutch/branches/2.x/src/java/org/apache/nutch/storage/WebPage.java
Modified: nutch/branches/2.x/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Sun Mar 24 14:08:05 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1533 Implement getPrevModifiedTime(), setPrevModifiedTime(),
getBatchId() and setBatchId() accessors in o.a.n.storage.WebPage (Feng via
lewismc)
+
* NUTCH-XX fix Elastic Search Ivy configuration (Binoy d via lewismc)
* NUTCH-1542 "adddays" param for generator not present in 2.x (tejasp)
Modified: nutch/branches/2.x/conf/gora-accumulo-mapping.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/conf/gora-accumulo-mapping.xml?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/conf/gora-accumulo-mapping.xml (original)
+++ nutch/branches/2.x/conf/gora-accumulo-mapping.xml Sun Mar 24 14:08:05 2013
@@ -42,6 +42,8 @@
<field name="contentType" family="f" qualifier="typ"/>
<field name="protocolStatus" family="f" qualifier="prot"/>
<field name="modifiedTime" family="f" qualifier="mod"/>
+ <field name="prevModifiedTime" family="f" qualifier="pmod"/>
+ <field name="batchId" family="f" qualifier="bid"/>
<!-- parse fields -->
<field name="title" family="p" qualifier="t"/>
Modified: nutch/branches/2.x/conf/gora-cassandra-mapping.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/conf/gora-cassandra-mapping.xml?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/conf/gora-cassandra-mapping.xml (original)
+++ nutch/branches/2.x/conf/gora-cassandra-mapping.xml Sun Mar 24 14:08:05 2013
@@ -35,6 +35,8 @@
<field name="content" family="f" qualifier="cnt"/>
<field name="contentType" family="f" qualifier="typ"/>
<field name="modifiedTime" family="f" qualifier="mod"/>
+ <field name="prevModifiedTime" family="f" qualifier="pmod"/>
+ <field name="batchId" family="f" qualifier="bid"/>
<!-- parse fields -->
<field name="title" family="p" qualifier="t"/>
Modified: nutch/branches/2.x/conf/gora-hbase-mapping.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/conf/gora-hbase-mapping.xml?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/conf/gora-hbase-mapping.xml (original)
+++ nutch/branches/2.x/conf/gora-hbase-mapping.xml Sun Mar 24 14:08:05 2013
@@ -41,7 +41,9 @@
<field name="contentType" family="f" qualifier="typ"/>
<field name="protocolStatus" family="f" qualifier="prot"/>
<field name="modifiedTime" family="f" qualifier="mod"/>
-
+ <field name="prevModifiedTime" family="f" qualifier="pmod"/>
+ <field name="batchId" family="f" qualifier="bid"/>
+
<!-- parse fields -->
<field name="title" family="p" qualifier="t"/>
<field name="text" family="p" qualifier="c"/>
Modified: nutch/branches/2.x/conf/gora-sql-mapping.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/conf/gora-sql-mapping.xml?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/conf/gora-sql-mapping.xml (original)
+++ nutch/branches/2.x/conf/gora-sql-mapping.xml Sun Mar 24 14:08:05 2013
@@ -30,6 +30,8 @@
<field name="contentType" column="typ" length="32"/>
<field name="protocolStatus" column="protocolStatus"/>
<field name="modifiedTime" column="modifiedTime"/>
+ <field name="prevModifiedTime" column="prevModifiedTime"/>
+ <field name="batchId" column="batchId"/>
<!-- parse fields -->
<field name="title" column="title" length="512"/>
Modified: nutch/branches/2.x/src/gora/webpage.avsc
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/gora/webpage.avsc?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/src/gora/webpage.avsc (original)
+++ nutch/branches/2.x/src/gora/webpage.avsc Sun Mar 24 14:08:05 2013
@@ -9,6 +9,7 @@
{"name": "fetchInterval", "type": "int"},
{"name": "retriesSinceFetch", "type": "int"},
{"name": "modifiedTime", "type": "long"},
+ {"name": "prevModifiedTime", "type": "long"},
{"name": "protocolStatus", "type": {
"name": "ProtocolStatus",
"type": "record",
@@ -41,6 +42,7 @@
{"name": "outlinks", "type": {"type": "map", "values": "string"}},
{"name": "inlinks", "type": {"type": "map", "values": "string"}},
{"name": "markers", "type": {"type": "map", "values": "string"}},
- {"name": "metadata", "type": {"type": "map", "values": "bytes"}}
+ {"name": "metadata", "type": {"type": "map", "values": "bytes"}},
+ {"name": "batchId", "type": "string"}
]
}
Modified:
nutch/branches/2.x/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
(original)
+++
nutch/branches/2.x/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
Sun Mar 24 14:08:05 2013
@@ -105,6 +105,7 @@ public class AdaptiveFetchSchedule exten
if (interval > MAX_INTERVAL) interval = MAX_INTERVAL;
page.setFetchTime(refTime + interval * 1000L);
page.setModifiedTime(modifiedTime);
+ page.setPrevModifiedTime(prevModifiedTime);
}
Modified:
nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdateReducer.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdateReducer.java?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdateReducer.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/crawl/DbUpdateReducer.java Sun
Mar 24 14:08:05 2013
@@ -127,14 +127,15 @@ extends GoraReducer<UrlWithScore, NutchW
long fetchTime = page.getFetchTime();
long prevFetchTime = page.getPrevFetchTime();
long modifiedTime = page.getModifiedTime();
+ long prevModifiedTime = page.getPrevModifiedTime();
- schedule.setFetchSchedule(url, page, prevFetchTime, 0L,
+ schedule.setFetchSchedule(url, page, prevFetchTime, prevModifiedTime,
fetchTime, modifiedTime, modified);
if (maxInterval < page.getFetchInterval())
schedule.forceRefetch(url, page, false);
break;
case CrawlStatus.STATUS_RETRY:
- schedule.setPageRetrySchedule(url, page, 0L, 0L, page.getFetchTime());
+ schedule.setPageRetrySchedule(url, page, 0L,
page.getPrevModifiedTime(), page.getFetchTime());
if (page.getRetriesSinceFetch() < retryMax) {
page.setStatus(CrawlStatus.STATUS_UNFETCHED);
} else {
@@ -142,7 +143,7 @@ extends GoraReducer<UrlWithScore, NutchW
}
break;
case CrawlStatus.STATUS_GONE:
- schedule.setPageGoneSchedule(url, page, 0L, 0L, page.getFetchTime());
+ schedule.setPageGoneSchedule(url, page, 0L,
page.getPrevModifiedTime(), page.getFetchTime());
break;
}
}
Modified:
nutch/branches/2.x/src/java/org/apache/nutch/crawl/DefaultFetchSchedule.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/crawl/DefaultFetchSchedule.java?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/crawl/DefaultFetchSchedule.java
(original)
+++
nutch/branches/2.x/src/java/org/apache/nutch/crawl/DefaultFetchSchedule.java
Sun Mar 24 14:08:05 2013
@@ -37,6 +37,7 @@ public class DefaultFetchSchedule extend
fetchTime, modifiedTime, state);
page.setFetchTime(fetchTime + page.getFetchInterval() * 1000L);
page.setModifiedTime(modifiedTime);
+ page.setPrevModifiedTime(prevModifiedTime);
}
}
Modified:
nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorReducer.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorReducer.java?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorReducer.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorReducer.java
Sun Mar 24 14:08:05 2013
@@ -74,6 +74,7 @@ extends GoraReducer<SelectorEntry, WebPa
}
Mark.GENERATE_MARK.putMark(page, batchId);
+ page.setBatchId(batchId);
try {
context.write(TableUtil.reverseUrl(key.url), page);
} catch (MalformedURLException e) {
Modified: nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java Sun
Mar 24 14:08:05 2013
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************************/
+
package org.apache.nutch.storage;
import java.nio.ByteBuffer;
@@ -26,6 +27,7 @@ import org.apache.avro.Protocol;
import org.apache.avro.util.Utf8;
import org.apache.avro.ipc.AvroRemoteException;
import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
import org.apache.avro.specific.SpecificExceptionBase;
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
Modified:
nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java
Sun Mar 24 14:08:05 2013
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************************/
+
package org.apache.nutch.storage;
import java.nio.ByteBuffer;
@@ -26,6 +27,7 @@ import org.apache.avro.Protocol;
import org.apache.avro.util.Utf8;
import org.apache.avro.ipc.AvroRemoteException;
import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
import org.apache.avro.specific.SpecificExceptionBase;
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
Modified: nutch/branches/2.x/src/java/org/apache/nutch/storage/WebPage.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/WebPage.java?rev=1460380&r1=1460379&r2=1460380&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/storage/WebPage.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/storage/WebPage.java Sun Mar
24 14:08:05 2013
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************************/
+
package org.apache.nutch.storage;
import java.nio.ByteBuffer;
@@ -26,6 +27,7 @@ import org.apache.avro.Protocol;
import org.apache.avro.util.Utf8;
import org.apache.avro.ipc.AvroRemoteException;
import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
import org.apache.avro.specific.SpecificExceptionBase;
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
@@ -38,7 +40,7 @@ import org.apache.gora.persistency.ListG
@SuppressWarnings("all")
public class WebPage extends PersistentBase {
- public static final Schema _SCHEMA =
Schema.parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"baseUrl\",\"type\":\"string\"},{\"name\":\"status\",\"type\":\"int\"},{\"name\":\"fetchTime\",\"type\":\"long\"},{\"name\":\"prevFetchTime\",\"type\":\"long\"},{\"name\":\"fetchInterval\",\"type\":\"int\"},{\"name\":\"retriesSinceFetch\",\"type\":\"int\"},{\"name\":\"modifiedTime\",\"type\":\"long\"},{\"name\":\"protocolStatus\",\"type\":{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"fields\":[{\"name\":\"code\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"lastModified\",\"type\":\"long\"}]}},{\"name\":\"content\",\"type\":\"bytes\"},{\"name\":\"contentType\",\"type\":\"string\"},{\"name\":\"prevSignature\",\"type\":\"bytes\"},{\"name\":\"signature\",\"type\":\"bytes\"},{\"name\":\"title\",\"type\":\"string\"},{\"name\":\"text\",\"type\":\"string\"},{\"name
\":\"parseStatus\",\"type\":{\"type\":\"record\",\"name\":\"ParseStatus\",\"fields\":[{\"name\":\"majorCode\",\"type\":\"int\"},{\"name\":\"minorCode\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}}]}},{\"name\":\"score\",\"type\":\"float\"},{\"name\":\"reprUrl\",\"type\":\"string\"},{\"name\":\"headers\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"inlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"markers\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":\"bytes\"}}]}");
+ public static final Schema _SCHEMA =
Schema.parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"baseUrl\",\"type\":\"string\"},{\"name\":\"status\",\"type\":\"int\"},{\"name\":\"fetchTime\",\"type\":\"long\"},{\"name\":\"prevFetchTime\",\"type\":\"long\"},{\"name\":\"fetchInterval\",\"type\":\"int\"},{\"name\":\"retriesSinceFetch\",\"type\":\"int\"},{\"name\":\"modifiedTime\",\"type\":\"long\"},{\"name\":\"prevModifiedTime\",\"type\":\"long\"},{\"name\":\"protocolStatus\",\"type\":{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"fields\":[{\"name\":\"code\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"lastModified\",\"type\":\"long\"}]}},{\"name\":\"content\",\"type\":\"bytes\"},{\"name\":\"contentType\",\"type\":\"string\"},{\"name\":\"prevSignature\",\"type\":\"bytes\"},{\"name\":\"signature\",\"type\":\"bytes\"},{\"name\":\"title\",\"type\":\"string\
"},{\"name\":\"text\",\"type\":\"string\"},{\"name\":\"parseStatus\",\"type\":{\"type\":\"record\",\"name\":\"ParseStatus\",\"fields\":[{\"name\":\"majorCode\",\"type\":\"int\"},{\"name\":\"minorCode\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}}]}},{\"name\":\"score\",\"type\":\"float\"},{\"name\":\"reprUrl\",\"type\":\"string\"},{\"name\":\"headers\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"inlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"markers\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":\"bytes\"}},{\"name\":\"batchId\",\"type\":\"string\"}]}");
public static enum Field {
BASE_URL(0,"baseUrl"),
STATUS(1,"status"),
@@ -47,21 +49,23 @@ public class WebPage extends PersistentB
FETCH_INTERVAL(4,"fetchInterval"),
RETRIES_SINCE_FETCH(5,"retriesSinceFetch"),
MODIFIED_TIME(6,"modifiedTime"),
- PROTOCOL_STATUS(7,"protocolStatus"),
- CONTENT(8,"content"),
- CONTENT_TYPE(9,"contentType"),
- PREV_SIGNATURE(10,"prevSignature"),
- SIGNATURE(11,"signature"),
- TITLE(12,"title"),
- TEXT(13,"text"),
- PARSE_STATUS(14,"parseStatus"),
- SCORE(15,"score"),
- REPR_URL(16,"reprUrl"),
- HEADERS(17,"headers"),
- OUTLINKS(18,"outlinks"),
- INLINKS(19,"inlinks"),
- MARKERS(20,"markers"),
- METADATA(21,"metadata"),
+ PREV_MODIFIED_TIME(7,"prevModifiedTime"),
+ PROTOCOL_STATUS(8,"protocolStatus"),
+ CONTENT(9,"content"),
+ CONTENT_TYPE(10,"contentType"),
+ PREV_SIGNATURE(11,"prevSignature"),
+ SIGNATURE(12,"signature"),
+ TITLE(13,"title"),
+ TEXT(14,"text"),
+ PARSE_STATUS(15,"parseStatus"),
+ SCORE(16,"score"),
+ REPR_URL(17,"reprUrl"),
+ HEADERS(18,"headers"),
+ OUTLINKS(19,"outlinks"),
+ INLINKS(20,"inlinks"),
+ MARKERS(21,"markers"),
+ METADATA(22,"metadata"),
+ BATCH_ID(23,"batchId"),
;
private int index;
private String name;
@@ -70,7 +74,7 @@ public class WebPage extends PersistentB
public String getName() {return name;}
public String toString() {return name;}
};
- public static final String[] _ALL_FIELDS =
{"baseUrl","status","fetchTime","prevFetchTime","fetchInterval","retriesSinceFetch","modifiedTime","protocolStatus","content","contentType","prevSignature","signature","title","text","parseStatus","score","reprUrl","headers","outlinks","inlinks","markers","metadata",};
+ public static final String[] _ALL_FIELDS =
{"baseUrl","status","fetchTime","prevFetchTime","fetchInterval","retriesSinceFetch","modifiedTime","prevModifiedTime","protocolStatus","content","contentType","prevSignature","signature","title","text","parseStatus","score","reprUrl","headers","outlinks","inlinks","markers","metadata","batchId",};
static {
PersistentBase.registerFields(WebPage.class, _ALL_FIELDS);
}
@@ -81,6 +85,7 @@ public class WebPage extends PersistentB
private int fetchInterval;
private int retriesSinceFetch;
private long modifiedTime;
+ private long prevModifiedTime;
private ProtocolStatus protocolStatus;
private ByteBuffer content;
private Utf8 contentType;
@@ -96,6 +101,7 @@ public class WebPage extends PersistentB
private Map<Utf8,Utf8> inlinks;
private Map<Utf8,Utf8> markers;
private Map<Utf8,ByteBuffer> metadata;
+ private Utf8 batchId;
public WebPage() {
this(new StateManagerImpl());
}
@@ -120,21 +126,23 @@ public class WebPage extends PersistentB
case 4: return fetchInterval;
case 5: return retriesSinceFetch;
case 6: return modifiedTime;
- case 7: return protocolStatus;
- case 8: return content;
- case 9: return contentType;
- case 10: return prevSignature;
- case 11: return signature;
- case 12: return title;
- case 13: return text;
- case 14: return parseStatus;
- case 15: return score;
- case 16: return reprUrl;
- case 17: return headers;
- case 18: return outlinks;
- case 19: return inlinks;
- case 20: return markers;
- case 21: return metadata;
+ case 7: return prevModifiedTime;
+ case 8: return protocolStatus;
+ case 9: return content;
+ case 10: return contentType;
+ case 11: return prevSignature;
+ case 12: return signature;
+ case 13: return title;
+ case 14: return text;
+ case 15: return parseStatus;
+ case 16: return score;
+ case 17: return reprUrl;
+ case 18: return headers;
+ case 19: return outlinks;
+ case 20: return inlinks;
+ case 21: return markers;
+ case 22: return metadata;
+ case 23: return batchId;
default: throw new AvroRuntimeException("Bad index");
}
}
@@ -150,21 +158,23 @@ public class WebPage extends PersistentB
case 4:fetchInterval = (Integer)_value; break;
case 5:retriesSinceFetch = (Integer)_value; break;
case 6:modifiedTime = (Long)_value; break;
- case 7:protocolStatus = (ProtocolStatus)_value; break;
- case 8:content = (ByteBuffer)_value; break;
- case 9:contentType = (Utf8)_value; break;
- case 10:prevSignature = (ByteBuffer)_value; break;
- case 11:signature = (ByteBuffer)_value; break;
- case 12:title = (Utf8)_value; break;
- case 13:text = (Utf8)_value; break;
- case 14:parseStatus = (ParseStatus)_value; break;
- case 15:score = (Float)_value; break;
- case 16:reprUrl = (Utf8)_value; break;
- case 17:headers = (Map<Utf8,Utf8>)_value; break;
- case 18:outlinks = (Map<Utf8,Utf8>)_value; break;
- case 19:inlinks = (Map<Utf8,Utf8>)_value; break;
- case 20:markers = (Map<Utf8,Utf8>)_value; break;
- case 21:metadata = (Map<Utf8,ByteBuffer>)_value; break;
+ case 7:prevModifiedTime = (Long)_value; break;
+ case 8:protocolStatus = (ProtocolStatus)_value; break;
+ case 9:content = (ByteBuffer)_value; break;
+ case 10:contentType = (Utf8)_value; break;
+ case 11:prevSignature = (ByteBuffer)_value; break;
+ case 12:signature = (ByteBuffer)_value; break;
+ case 13:title = (Utf8)_value; break;
+ case 14:text = (Utf8)_value; break;
+ case 15:parseStatus = (ParseStatus)_value; break;
+ case 16:score = (Float)_value; break;
+ case 17:reprUrl = (Utf8)_value; break;
+ case 18:headers = (Map<Utf8,Utf8>)_value; break;
+ case 19:outlinks = (Map<Utf8,Utf8>)_value; break;
+ case 20:inlinks = (Map<Utf8,Utf8>)_value; break;
+ case 21:markers = (Map<Utf8,Utf8>)_value; break;
+ case 22:metadata = (Map<Utf8,ByteBuffer>)_value; break;
+ case 23:batchId = (Utf8)_value; break;
default: throw new AvroRuntimeException("Bad index");
}
}
@@ -210,144 +220,156 @@ public class WebPage extends PersistentB
public void setModifiedTime(long value) {
put(6, value);
}
+ public long getPrevModifiedTime() {
+ return (Long) get(7);
+ }
+ public void setPrevModifiedTime(long value) {
+ put(7, value);
+ }
public ProtocolStatus getProtocolStatus() {
- return (ProtocolStatus) get(7);
+ return (ProtocolStatus) get(8);
}
public void setProtocolStatus(ProtocolStatus value) {
- put(7, value);
+ put(8, value);
}
public ByteBuffer getContent() {
- return (ByteBuffer) get(8);
+ return (ByteBuffer) get(9);
}
public void setContent(ByteBuffer value) {
- put(8, value);
+ put(9, value);
}
public Utf8 getContentType() {
- return (Utf8) get(9);
+ return (Utf8) get(10);
}
public void setContentType(Utf8 value) {
- put(9, value);
+ put(10, value);
}
public ByteBuffer getPrevSignature() {
- return (ByteBuffer) get(10);
+ return (ByteBuffer) get(11);
}
public void setPrevSignature(ByteBuffer value) {
- put(10, value);
+ put(11, value);
}
public ByteBuffer getSignature() {
- return (ByteBuffer) get(11);
+ return (ByteBuffer) get(12);
}
public void setSignature(ByteBuffer value) {
- put(11, value);
+ put(12, value);
}
public Utf8 getTitle() {
- return (Utf8) get(12);
+ return (Utf8) get(13);
}
public void setTitle(Utf8 value) {
- put(12, value);
+ put(13, value);
}
public Utf8 getText() {
- return (Utf8) get(13);
+ return (Utf8) get(14);
}
public void setText(Utf8 value) {
- put(13, value);
+ put(14, value);
}
public ParseStatus getParseStatus() {
- return (ParseStatus) get(14);
+ return (ParseStatus) get(15);
}
public void setParseStatus(ParseStatus value) {
- put(14, value);
+ put(15, value);
}
public float getScore() {
- return (Float) get(15);
+ return (Float) get(16);
}
public void setScore(float value) {
- put(15, value);
+ put(16, value);
}
public Utf8 getReprUrl() {
- return (Utf8) get(16);
+ return (Utf8) get(17);
}
public void setReprUrl(Utf8 value) {
- put(16, value);
+ put(17, value);
}
public Map<Utf8, Utf8> getHeaders() {
- return (Map<Utf8, Utf8>) get(17);
+ return (Map<Utf8, Utf8>) get(18);
}
public Utf8 getFromHeaders(Utf8 key) {
if (headers == null) { return null; }
return headers.get(key);
}
public void putToHeaders(Utf8 key, Utf8 value) {
- getStateManager().setDirty(this, 17);
+ getStateManager().setDirty(this, 18);
headers.put(key, value);
}
public Utf8 removeFromHeaders(Utf8 key) {
if (headers == null) { return null; }
- getStateManager().setDirty(this, 17);
+ getStateManager().setDirty(this, 18);
return headers.remove(key);
}
public Map<Utf8, Utf8> getOutlinks() {
- return (Map<Utf8, Utf8>) get(18);
+ return (Map<Utf8, Utf8>) get(19);
}
public Utf8 getFromOutlinks(Utf8 key) {
if (outlinks == null) { return null; }
return outlinks.get(key);
}
public void putToOutlinks(Utf8 key, Utf8 value) {
- getStateManager().setDirty(this, 18);
+ getStateManager().setDirty(this, 19);
outlinks.put(key, value);
}
public Utf8 removeFromOutlinks(Utf8 key) {
if (outlinks == null) { return null; }
- getStateManager().setDirty(this, 18);
+ getStateManager().setDirty(this, 19);
return outlinks.remove(key);
}
public Map<Utf8, Utf8> getInlinks() {
- return (Map<Utf8, Utf8>) get(19);
+ return (Map<Utf8, Utf8>) get(20);
}
public Utf8 getFromInlinks(Utf8 key) {
if (inlinks == null) { return null; }
return inlinks.get(key);
}
public void putToInlinks(Utf8 key, Utf8 value) {
- getStateManager().setDirty(this, 19);
+ getStateManager().setDirty(this, 20);
inlinks.put(key, value);
}
public Utf8 removeFromInlinks(Utf8 key) {
if (inlinks == null) { return null; }
- getStateManager().setDirty(this, 19);
+ getStateManager().setDirty(this, 20);
return inlinks.remove(key);
}
public Map<Utf8, Utf8> getMarkers() {
- return (Map<Utf8, Utf8>) get(20);
+ return (Map<Utf8, Utf8>) get(21);
}
public Utf8 getFromMarkers(Utf8 key) {
if (markers == null) { return null; }
return markers.get(key);
}
public void putToMarkers(Utf8 key, Utf8 value) {
- getStateManager().setDirty(this, 20);
+ getStateManager().setDirty(this, 21);
markers.put(key, value);
}
public Utf8 removeFromMarkers(Utf8 key) {
if (markers == null) { return null; }
- getStateManager().setDirty(this, 20);
+ getStateManager().setDirty(this, 21);
return markers.remove(key);
}
public Map<Utf8, ByteBuffer> getMetadata() {
- return (Map<Utf8, ByteBuffer>) get(21);
+ return (Map<Utf8, ByteBuffer>) get(22);
}
public ByteBuffer getFromMetadata(Utf8 key) {
if (metadata == null) { return null; }
return metadata.get(key);
}
public void putToMetadata(Utf8 key, ByteBuffer value) {
- getStateManager().setDirty(this, 21);
+ getStateManager().setDirty(this, 22);
metadata.put(key, value);
}
public ByteBuffer removeFromMetadata(Utf8 key) {
if (metadata == null) { return null; }
- getStateManager().setDirty(this, 21);
+ getStateManager().setDirty(this, 22);
return metadata.remove(key);
}
+ public Utf8 getBatchId() {
+ return (Utf8) get(23);
+ }
+ public void setBatchId(Utf8 value) {
+ put(23, value);
+ }
}
