Modified: nutch/branches/2.x/src/java/org/apache/nutch/storage/Mark.java URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/Mark.java?rev=1594812&r1=1594811&r2=1594812&view=diff ============================================================================== --- nutch/branches/2.x/src/java/org/apache/nutch/storage/Mark.java (original) +++ nutch/branches/2.x/src/java/org/apache/nutch/storage/Mark.java Thu May 15 08:10:07 2014 @@ -29,7 +29,7 @@ public enum Mark { } public void putMark(WebPage page, Utf8 markValue) { - page.putToMarkers(name, markValue); + page.getMarkers().put(name, markValue); } public void putMark(WebPage page, String markValue) { @@ -37,11 +37,11 @@ public enum Mark { } public Utf8 removeMark(WebPage page) { - return page.removeFromMarkers(name); + return (Utf8) page.getMarkers().put(name, null); } public Utf8 checkMark(WebPage page) { - return page.getFromMarkers(name); + return (Utf8) page.getMarkers().get(name); } /** @@ -50,8 +50,8 @@ public enum Mark { * @return If the mark was present. */ public Utf8 removeMarkIfExist(WebPage page) { - if (page.getFromMarkers(name) != null) { - return page.removeFromMarkers(name); + if (checkMark(page) != null) { + return removeMark(page); } return null; }
Modified: nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java?rev=1594812&r1=1594811&r2=1594812&view=diff ============================================================================== --- nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java (original) +++ nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java Thu May 15 08:10:07 2014 @@ -1,113 +1,416 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at +/** + * Autogenerated by Avro * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package org.apache.nutch.storage; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.HashMap; -import org.apache.avro.Protocol; -import org.apache.avro.Schema; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Protocol; -import org.apache.avro.util.Utf8; -import org.apache.avro.ipc.AvroRemoteException; -import org.apache.avro.generic.GenericArray; -import org.apache.avro.specific.FixedSize; -import org.apache.avro.specific.SpecificExceptionBase; -import org.apache.avro.specific.SpecificRecordBase; -import org.apache.avro.specific.SpecificRecord; -import org.apache.avro.specific.SpecificFixed; -import org.apache.gora.persistency.StateManager; -import org.apache.gora.persistency.impl.PersistentBase; -import org.apache.gora.persistency.impl.StateManagerImpl; -import org.apache.gora.persistency.StatefulHashMap; -import org.apache.gora.persistency.ListGenericArray; - + * DO NOT EDIT DIRECTLY + */ +package org.apache.nutch.storage; @SuppressWarnings("all") -public class ParseStatus extends PersistentBase { - public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"ParseStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"majorCode\",\"type\":\"int\"},{\"name\":\"minorCode\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}}]}"); +public class ParseStatus extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ParseStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"majorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"minorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]}]}"); + + /** Enum containing all data bean's fields. */ public static enum Field { - MAJOR_CODE(0,"majorCode"), - MINOR_CODE(1,"minorCode"), - ARGS(2,"args"), + __G__DIRTY(0, "__g__dirty"), + MAJOR_CODE(1, "majorCode"), + MINOR_CODE(2, "minorCode"), + ARGS(3, "args"), ; + /** + * Field's index. + */ private int index; + + /** + * Field's name. + */ private String name; + + /** + * Field's constructor + * @param index field's index. + * @param name field's name. + */ Field(int index, String name) {this.index=index;this.name=name;} + + /** + * Gets field's index. + * @return int field's index. + */ public int getIndex() {return index;} + + /** + * Gets field's name. + * @return String field's name. + */ public String getName() {return name;} + + /** + * Gets field's attributes to string. + * @return String field's attributes to string. + */ public String toString() {return name;} }; - public static final String[] _ALL_FIELDS = {"majorCode","minorCode","args",}; - static { - PersistentBase.registerFields(ParseStatus.class, _ALL_FIELDS); - } + + public static final String[] _ALL_FIELDS = { + "__g__dirty", + "majorCode", + "minorCode", + "args", + }; + + /** Bytes used to represent weather or not a field is dirty. */ + private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[1]); private int majorCode; private int minorCode; - private GenericArray<Utf8> args; - public ParseStatus() { - this(new StateManagerImpl()); - } - public ParseStatus(StateManager stateManager) { - super(stateManager); - args = new ListGenericArray<Utf8>(getSchema().getField("args").schema()); - } - public ParseStatus newInstance(StateManager stateManager) { - return new ParseStatus(stateManager); - } - public Schema getSchema() { return _SCHEMA; } - public Object get(int _field) { - switch (_field) { - case 0: return majorCode; - case 1: return minorCode; - case 2: return args; - default: throw new AvroRuntimeException("Bad index"); + private java.util.List<CharSequence> args; + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public Object get(int field$) { + switch (field$) { + case 0: return __g__dirty; + case 1: return majorCode; + case 2: return minorCode; + case 3: return args; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } + + // Used by DatumReader. Applications should not call. @SuppressWarnings(value="unchecked") - public void put(int _field, Object _value) { - if(isFieldEqual(_field, _value)) return; - getStateManager().setDirty(this, _field); - switch (_field) { - case 0:majorCode = (Integer)_value; break; - case 1:minorCode = (Integer)_value; break; - case 2:args = (GenericArray<Utf8>)_value; break; - default: throw new AvroRuntimeException("Bad index"); + public void put(int field$, Object value) { + switch (field$) { + case 0: __g__dirty = (java.nio.ByteBuffer)(value); break; + case 1: majorCode = (Integer)(value); break; + case 2: minorCode = (Integer)(value); break; + case 3: args = (java.util.List<CharSequence>)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)value)); break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } - public int getMajorCode() { - return (Integer) get(0); + + /** + * Gets the value of the 'majorCode' field. + */ + public Integer getMajorCode() { + return majorCode; + } + + /** + * Sets the value of the 'majorCode' field. + * @param value the value to set. + */ + public void setMajorCode(Integer value) { + this.majorCode = value; + setDirty(1); + } + + /** + * Checks the dirty status of the 'majorCode' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMajorCodeDirty(Integer value) { + return isDirty(1); + } + + /** + * Gets the value of the 'minorCode' field. + */ + public Integer getMinorCode() { + return minorCode; } - public void setMajorCode(int value) { - put(0, value); + + /** + * Sets the value of the 'minorCode' field. + * @param value the value to set. + */ + public void setMinorCode(Integer value) { + this.minorCode = value; + setDirty(2); + } + + /** + * Checks the dirty status of the 'minorCode' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMinorCodeDirty(Integer value) { + return isDirty(2); + } + + /** + * Gets the value of the 'args' field. + */ + public java.util.List<CharSequence> getArgs() { + return args; + } + + /** + * Sets the value of the 'args' field. + * @param value the value to set. + */ + public void setArgs(java.util.List<CharSequence> value) { + this.args = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper(value); + setDirty(3); + } + + /** + * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isArgsDirty(java.util.List<CharSequence> value) { + return isDirty(3); } - public int getMinorCode() { - return (Integer) get(1); + + /** Creates a new ParseStatus RecordBuilder */ + public static Builder newBuilder() { + return new Builder(); + } + + /** Creates a new ParseStatus RecordBuilder by copying an existing Builder */ + public static Builder newBuilder(Builder other) { + return new Builder(other); + } + + /** Creates a new ParseStatus RecordBuilder by copying an existing ParseStatus instance */ + public static Builder newBuilder(ParseStatus other) { + return new Builder(other); + } + + private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer( + java.nio.ByteBuffer input) { + java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity()); + int position = input.position(); + input.reset(); + int mark = input.position(); + int limit = input.limit(); + input.rewind(); + input.limit(input.capacity()); + copy.put(input); + input.rewind(); + copy.rewind(); + input.position(mark); + input.mark(); + copy.position(mark); + copy.mark(); + input.position(position); + copy.position(position); + input.limit(limit); + copy.limit(limit); + return copy.asReadOnlyBuffer(); + } + + /** + * RecordBuilder for ParseStatus instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<ParseStatus> + implements org.apache.avro.data.RecordBuilder<ParseStatus> { + + private java.nio.ByteBuffer __g__dirty; + private int majorCode; + private int minorCode; + private java.util.List<CharSequence> args; + + /** Creates a new Builder */ + private Builder() { + super(ParseStatus.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing ParseStatus instance */ + private Builder(ParseStatus other) { + super(ParseStatus.SCHEMA$); + if (isValidValue(fields()[0], other.__g__dirty)) { + this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.majorCode)) { + this.majorCode = (Integer) data().deepCopy(fields()[1].schema(), other.majorCode); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.minorCode)) { + this.minorCode = (Integer) data().deepCopy(fields()[2].schema(), other.minorCode); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.args)) { + this.args = (java.util.List<CharSequence>) data().deepCopy(fields()[3].schema(), other.args); + fieldSetFlags()[3] = true; + } + } + + /** Gets the value of the 'majorCode' field */ + public Integer getMajorCode() { + return majorCode; + } + + /** Sets the value of the 'majorCode' field */ + public Builder setMajorCode(int value) { + validate(fields()[1], value); + this.majorCode = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'majorCode' field has been set */ + public boolean hasMajorCode() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'majorCode' field */ + public Builder clearMajorCode() { + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'minorCode' field */ + public Integer getMinorCode() { + return minorCode; + } + + /** Sets the value of the 'minorCode' field */ + public Builder setMinorCode(int value) { + validate(fields()[2], value); + this.minorCode = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'minorCode' field has been set */ + public boolean hasMinorCode() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'minorCode' field */ + public Builder clearMinorCode() { + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'args' field */ + public java.util.List<CharSequence> getArgs() { + return args; + } + + /** Sets the value of the 'args' field */ + public Builder setArgs(java.util.List<CharSequence> value) { + validate(fields()[3], value); + this.args = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'args' field has been set */ + public boolean hasArgs() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'args' field */ + public Builder clearArgs() { + args = null; + fieldSetFlags()[3] = false; + return this; + } + + @Override + public ParseStatus build() { + try { + ParseStatus record = new ParseStatus(); + record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[1]); + record.majorCode = fieldSetFlags()[1] ? this.majorCode : (Integer) defaultValue(fields()[1]); + record.minorCode = fieldSetFlags()[2] ? this.minorCode : (Integer) defaultValue(fields()[2]); + record.args = fieldSetFlags()[3] ? this.args : (java.util.List<CharSequence>) new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)defaultValue(fields()[3])); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } } - public void setMinorCode(int value) { - put(1, value); + + public Tombstone getTombstone(){ + return TOMBSTONE; } - @SuppressWarnings("unchecked") - public GenericArray<Utf8> getArgs() { - return (GenericArray<Utf8>) get(2); + + public ParseStatus newInstance(){ + return newBuilder().build(); } - public void addToArgs(Utf8 element) { - getStateManager().setDirty(this, 2); - args.add(element); + + private static final Tombstone TOMBSTONE = new Tombstone(); + + public static final class Tombstone extends ParseStatus implements org.apache.gora.persistency.Tombstone { + + private Tombstone() { } + + /** + * Gets the value of the 'majorCode' field. + */ + public Integer getMajorCode() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'majorCode' field. + * @param value the value to set. + */ + public void setMajorCode(Integer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'majorCode' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMajorCodeDirty(Integer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'minorCode' field. + */ + public Integer getMinorCode() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'minorCode' field. + * @param value the value to set. + */ + public void setMinorCode(Integer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'minorCode' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMinorCodeDirty(Integer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'args' field. + */ + public java.util.List<CharSequence> getArgs() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'args' field. + * @param value the value to set. + */ + public void setArgs(java.util.List<CharSequence> value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isArgsDirty(java.util.List<CharSequence> value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + } -} + +} \ No newline at end of file Modified: nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java?rev=1594812&r1=1594811&r2=1594812&view=diff ============================================================================== --- nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java (original) +++ nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java Thu May 15 08:10:07 2014 @@ -1,122 +1,428 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at +/** + * Autogenerated by Avro * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package org.apache.nutch.storage; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.HashMap; -import org.apache.avro.Protocol; -import org.apache.avro.Schema; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Protocol; -import org.apache.avro.util.Utf8; -import org.apache.avro.ipc.AvroRemoteException; -import org.apache.avro.generic.GenericArray; -import org.apache.avro.specific.FixedSize; -import org.apache.avro.specific.SpecificExceptionBase; -import org.apache.avro.specific.SpecificRecordBase; -import org.apache.avro.specific.SpecificRecord; -import org.apache.avro.specific.SpecificFixed; -import org.apache.gora.persistency.StateManager; -import org.apache.gora.persistency.impl.PersistentBase; -import org.apache.gora.persistency.impl.StateManagerImpl; -import org.apache.gora.persistency.StatefulHashMap; -import org.apache.gora.persistency.ListGenericArray; + * DO NOT EDIT DIRECTLY + */ +package org.apache.nutch.storage; + import org.apache.nutch.protocol.ProtocolStatusUtils; @SuppressWarnings("all") -public class ProtocolStatus extends PersistentBase { - public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"code\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"lastModified\",\"type\":\"long\"}]}"); +public class ProtocolStatus extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"code\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]},{\"name\":\"lastModified\",\"type\":\"long\",\"default\":0}]}"); + + /** Enum containing all data bean's fields. */ public static enum Field { - CODE(0,"code"), - ARGS(1,"args"), - LAST_MODIFIED(2,"lastModified"), + __G__DIRTY(0, "__g__dirty"), + CODE(1, "code"), + ARGS(2, "args"), + LAST_MODIFIED(3, "lastModified"), ; + /** + * Field's index. + */ private int index; + + /** + * Field's name. + */ private String name; + + /** + * Field's constructor + * @param index field's index. + * @param name field's name. + */ Field(int index, String name) {this.index=index;this.name=name;} + + /** + * Gets field's index. + * @return int field's index. + */ public int getIndex() {return index;} + + /** + * Gets field's name. + * @return String field's name. + */ public String getName() {return name;} + + /** + * Gets field's attributes to string. + * @return String field's attributes to string. + */ public String toString() {return name;} }; - public static final String[] _ALL_FIELDS = {"code","args","lastModified",}; - static { - PersistentBase.registerFields(ProtocolStatus.class, _ALL_FIELDS); - } + + public static final String[] _ALL_FIELDS = { + "__g__dirty", + "code", + "args", + "lastModified", + }; + + /** Bytes used to represent weather or not a field is dirty. */ + private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[1]); private int code; - private GenericArray<Utf8> args; + private java.util.List<CharSequence> args; private long lastModified; - public ProtocolStatus() { - this(new StateManagerImpl()); - } - public ProtocolStatus(StateManager stateManager) { - super(stateManager); - args = new ListGenericArray<Utf8>(getSchema().getField("args").schema()); - } - public ProtocolStatus newInstance(StateManager stateManager) { - return new ProtocolStatus(stateManager); - } - public Schema getSchema() { return _SCHEMA; } - public Object get(int _field) { - switch (_field) { - case 0: return code; - case 1: return args; - case 2: return lastModified; - default: throw new AvroRuntimeException("Bad index"); + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public Object get(int field$) { + switch (field$) { + case 0: return __g__dirty; + case 1: return code; + case 2: return args; + case 3: return lastModified; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } + + // Used by DatumReader. Applications should not call. @SuppressWarnings(value="unchecked") - public void put(int _field, Object _value) { - if(isFieldEqual(_field, _value)) return; - getStateManager().setDirty(this, _field); - switch (_field) { - case 0:code = (Integer)_value; break; - case 1:args = (GenericArray<Utf8>)_value; break; - case 2:lastModified = (Long)_value; break; - default: throw new AvroRuntimeException("Bad index"); - } - } - public int getCode() { - return (Integer) get(0); - } - public void setCode(int value) { - put(0, value); - } - @SuppressWarnings("unchecked") - public GenericArray<Utf8> getArgs() { - return (GenericArray<Utf8>) get(1); - } - public void addToArgs(Utf8 element) { - getStateManager().setDirty(this, 1); - args.add(element); + public void put(int field$, Object value) { + switch (field$) { + case 0: __g__dirty = (java.nio.ByteBuffer)(value); break; + case 1: code = (Integer)(value); break; + case 2: args = (java.util.List<CharSequence>)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)value)); break; + case 3: lastModified = (Long)(value); break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } } - public long getLastModified() { - return (Long) get(2); + + /** + * Gets the value of the 'code' field. + */ + public Integer getCode() { + return code; } - public void setLastModified(long value) { - put(2, value); + + /** + * Sets the value of the 'code' field. + * @param value the value to set. + */ + public void setCode(Integer value) { + this.code = value; + setDirty(1); + } + + /** + * Checks the dirty status of the 'code' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isCodeDirty(Integer value) { + return isDirty(1); + } + + /** + * Gets the value of the 'args' field. + */ + public java.util.List<CharSequence> getArgs() { + return args; + } + + /** + * Sets the value of the 'args' field. + * @param value the value to set. + */ + public void setArgs(java.util.List<CharSequence> value) { + this.args = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper(value); + setDirty(2); + } + + /** + * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isArgsDirty(java.util.List<CharSequence> value) { + return isDirty(2); + } + + /** + * Gets the value of the 'lastModified' field. + */ + public Long getLastModified() { + return lastModified; + } + + /** + * Sets the value of the 'lastModified' field. + * @param value the value to set. + */ + public void setLastModified(Long value) { + this.lastModified = value; + setDirty(3); } /** + * Checks the dirty status of the 'lastModified' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isLastModifiedDirty(Long value) { + return isDirty(3); + } + + /** * A convenience method which returns a successful {@link ProtocolStatus}. + * * @return the {@link ProtocolStatus} value for 200 (success). */ public boolean isSuccess() { - return code == ProtocolStatusUtils.SUCCESS; + return code == ProtocolStatusUtils.SUCCESS; + } + + /** Creates a new ProtocolStatus RecordBuilder */ + public static Builder newBuilder() { + return new Builder(); + } + + /** Creates a new ProtocolStatus RecordBuilder by copying an existing Builder */ + public static Builder newBuilder(Builder other) { + return new Builder(other); + } + + /** Creates a new ProtocolStatus RecordBuilder by copying an existing ProtocolStatus instance */ + public static Builder newBuilder(ProtocolStatus other) { + return new Builder(other); + } + + private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer( + java.nio.ByteBuffer input) { + java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity()); + int position = input.position(); + input.reset(); + int mark = input.position(); + int limit = input.limit(); + input.rewind(); + input.limit(input.capacity()); + copy.put(input); + input.rewind(); + copy.rewind(); + input.position(mark); + input.mark(); + copy.position(mark); + copy.mark(); + input.position(position); + copy.position(position); + input.limit(limit); + copy.limit(limit); + return copy.asReadOnlyBuffer(); } -} + + /** + * RecordBuilder for ProtocolStatus instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<ProtocolStatus> + implements org.apache.avro.data.RecordBuilder<ProtocolStatus> { + + private java.nio.ByteBuffer __g__dirty; + private int code; + private java.util.List<CharSequence> args; + private long lastModified; + + /** Creates a new Builder */ + private Builder() { + super(ProtocolStatus.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing ProtocolStatus instance */ + private Builder(ProtocolStatus other) { + super(ProtocolStatus.SCHEMA$); + if (isValidValue(fields()[0], other.__g__dirty)) { + this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.code)) { + this.code = (Integer) data().deepCopy(fields()[1].schema(), other.code); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.args)) { + this.args = (java.util.List<CharSequence>) data().deepCopy(fields()[2].schema(), other.args); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.lastModified)) { + this.lastModified = (Long) data().deepCopy(fields()[3].schema(), other.lastModified); + fieldSetFlags()[3] = true; + } + } + + /** Gets the value of the 'code' field */ + public Integer getCode() { + return code; + } + + /** Sets the value of the 'code' field */ + public Builder setCode(int value) { + validate(fields()[1], value); + this.code = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'code' field has been set */ + public boolean hasCode() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'code' field */ + public Builder clearCode() { + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'args' field */ + public java.util.List<CharSequence> getArgs() { + return args; + } + + /** Sets the value of the 'args' field */ + public Builder setArgs(java.util.List<CharSequence> value) { + validate(fields()[2], value); + this.args = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'args' field has been set */ + public boolean hasArgs() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'args' field */ + public Builder clearArgs() { + args = null; + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'lastModified' field */ + public Long getLastModified() { + return lastModified; + } + + /** Sets the value of the 'lastModified' field */ + public Builder setLastModified(long value) { + validate(fields()[3], value); + this.lastModified = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'lastModified' field has been set */ + public boolean hasLastModified() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'lastModified' field */ + public Builder clearLastModified() { + fieldSetFlags()[3] = false; + return this; + } + + @Override + public ProtocolStatus build() { + try { + ProtocolStatus record = new ProtocolStatus(); + record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[1]); + record.code = fieldSetFlags()[1] ? this.code : (Integer) defaultValue(fields()[1]); + record.args = fieldSetFlags()[2] ? this.args : (java.util.List<CharSequence>) new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)defaultValue(fields()[2])); + record.lastModified = fieldSetFlags()[3] ? this.lastModified : (Long) defaultValue(fields()[3]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + + public Tombstone getTombstone(){ + return TOMBSTONE; + } + + public ProtocolStatus newInstance(){ + return newBuilder().build(); + } + + private static final Tombstone TOMBSTONE = new Tombstone(); + + public static final class Tombstone extends ProtocolStatus implements org.apache.gora.persistency.Tombstone { + + private Tombstone() { } + + /** + * Gets the value of the 'code' field. + */ + public Integer getCode() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'code' field. + * @param value the value to set. + */ + public void setCode(Integer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'code' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isCodeDirty(Integer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'args' field. + */ + public java.util.List<CharSequence> getArgs() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'args' field. + * @param value the value to set. + */ + public void setArgs(java.util.List<CharSequence> value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isArgsDirty(java.util.List<CharSequence> value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'lastModified' field. + */ + public Long getLastModified() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'lastModified' field. + * @param value the value to set. + */ + public void setLastModified(Long value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'lastModified' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isLastModifiedDirty(Long value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + + } + +} \ No newline at end of file Modified: nutch/branches/2.x/src/java/org/apache/nutch/storage/StorageUtils.java URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/StorageUtils.java?rev=1594812&r1=1594811&r2=1594812&view=diff ============================================================================== --- nutch/branches/2.x/src/java/org/apache/nutch/storage/StorageUtils.java (original) +++ nutch/branches/2.x/src/java/org/apache/nutch/storage/StorageUtils.java Thu May 15 08:10:07 2014 @@ -16,10 +16,6 @@ ******************************************************************************/ package org.apache.nutch.storage; -import java.io.IOException; -import java.util.Collection; -import java.util.Iterator; - import org.apache.gora.mapreduce.GoraMapper; import org.apache.gora.mapreduce.GoraOutputFormat; import org.apache.gora.mapreduce.GoraReducer; @@ -33,6 +29,10 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Partitioner; import org.apache.nutch.metadata.Nutch; +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; + /** * Entry point to Gora store/mapreduce functionality. * Translates the concept of "crawlid" to the corresponding Gora support. @@ -52,23 +52,23 @@ public class StorageUtils { @SuppressWarnings("unchecked") public static <K, V extends Persistent> DataStore<K, V> createWebStore(Configuration conf, Class<K> keyClass, Class<V> persistentClass) throws ClassNotFoundException, GoraException { - - String schema = null; + + String crawlId = conf.get(Nutch.CRAWL_ID_KEY, ""); + String schemaPrefix = ""; + if (!crawlId.isEmpty()) { + schemaPrefix = crawlId + "_"; + } + + String schema; if (WebPage.class.equals(persistentClass)) { schema = conf.get("storage.schema.webpage", "webpage"); + conf.set("preferred.schema.name", schemaPrefix + "webpage"); } else if (Host.class.equals(persistentClass)) { schema = conf.get("storage.schema.host", "host"); + conf.set("preferred.schema.name", schemaPrefix + "host"); } else { throw new UnsupportedOperationException("Unable to create store for class " + persistentClass); } - - String crawlId = conf.get(Nutch.CRAWL_ID_KEY, ""); - - if (!crawlId.isEmpty()) { - conf.set("schema.prefix", crawlId + "_"); - } else { - conf.set("schema.prefix", ""); - } Class<? extends DataStore<K, V>> dataStoreClass = (Class<? extends DataStore<K, V>>) getDataStoreClass(conf);
