Author: ferdy
Date: Thu Apr 19 12:15:35 2012
New Revision: 1327922
URL: http://svn.apache.org/viewvc?rev=1327922&view=rev
Log:
GORA-120 Dirty fields are not correctly applied after serialization and map
clearance
Modified:
gora/trunk/CHANGES.txt
gora/trunk/gora-core/src/main/java/org/apache/gora/avro/PersistentDatumReader.java
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/StatefulHashMap.java
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/StatefulMap.java
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
Modified: gora/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/gora/trunk/CHANGES.txt?rev=1327922&r1=1327921&r2=1327922&view=diff
==============================================================================
--- gora/trunk/CHANGES.txt (original)
+++ gora/trunk/CHANGES.txt Thu Apr 19 12:15:35 2012
@@ -7,6 +7,8 @@ Gora Change Log
0.2 Release: 20/04/2012
Jira Release Report:
https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12311172&version=12315541
+* GORA-120 Dirty fields are not correctly applied after serialization and map
clearance (ferdy)
+
* GORA-115 Flushing HBaseStore should flush all HTable instances. (ferdy)
* Make hbase autoflush default to false and make autoflush configurable rather
than hardcoded (stack via lewismc)
Modified:
gora/trunk/gora-core/src/main/java/org/apache/gora/avro/PersistentDatumReader.java
URL:
http://svn.apache.org/viewvc/gora/trunk/gora-core/src/main/java/org/apache/gora/avro/PersistentDatumReader.java?rev=1327922&r1=1327921&r2=1327922&view=diff
==============================================================================
---
gora/trunk/gora-core/src/main/java/org/apache/gora/avro/PersistentDatumReader.java
(original)
+++
gora/trunk/gora-core/src/main/java/org/apache/gora/avro/PersistentDatumReader.java
Thu Apr 19 12:15:35 2012
@@ -19,8 +19,10 @@
package org.apache.gora.avro;
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.WeakHashMap;
import org.apache.avro.Schema;
@@ -141,6 +143,9 @@ public class PersistentDatumReader<T ext
for (i = 0; i < dirtyFields.length; i++) {
if (dirtyFields[i]) {
persistent.setDirty(i);
+ }
+ else {
+ persistent.clearDirty(i);
}
}
return record;
@@ -163,26 +168,32 @@ public class PersistentDatumReader<T ext
@SuppressWarnings("unchecked")
protected Object readMap(Object old, Schema expected, ResolvingDecoder in)
throws IOException {
-
StatefulMap<Utf8, ?> map = (StatefulMap<Utf8, ?>) newMap(old, 0);
- map.clearStates();
+ Map<Utf8, State> tempStates = null;
if (readDirtyBits) {
+ tempStates = new HashMap<Utf8, State>();
int size = in.readInt();
for (int j = 0; j < size; j++) {
Utf8 key = in.readString(null);
State state = State.values()[in.readInt()];
- map.putState(key, state);
+ tempStates.put(key, state);
+ }
+ }
+ super.readMap(map, expected, in);
+ map.clearStates();
+ if (readDirtyBits) {
+ for (Entry<Utf8, State> entry : tempStates.entrySet()) {
+ map.putState(entry.getKey(), entry.getValue());
}
}
- return super.readMap(map, expected, in);
+ return map;
}
@Override
@SuppressWarnings({ "rawtypes" })
protected Object newMap(Object old, int size) {
if (old instanceof StatefulHashMap) {
- ((Map) old).clear();
- ((StatefulHashMap)old).clearStates();
+ ((StatefulHashMap)old).reuse();
return old;
}
return new StatefulHashMap<Object, Object>();
Modified:
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/StatefulHashMap.java
URL:
http://svn.apache.org/viewvc/gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/StatefulHashMap.java?rev=1327922&r1=1327921&r2=1327922&view=diff
==============================================================================
---
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/StatefulHashMap.java
(original)
+++
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/StatefulHashMap.java
Thu Apr 19 12:15:35 2012
@@ -29,31 +29,49 @@ public class StatefulHashMap<K, V> exten
*/
private Map<K, State> keyStates = new HashMap<K, State>();
+ /**
+ * Create an empty instance.
+ */
public StatefulHashMap() {
this(null);
}
+ /**
+ * Create an instance with initial entries. These entries are added
stateless;
+ * in other words the statemap will be clear after the construction.
+ *
+ * @param m The map with initial entries.
+ */
public StatefulHashMap(Map<K, V> m) {
super();
if (m == null) {
return;
}
- super.putAll(m);
+ for (java.util.Map.Entry<K, V> entry : m.entrySet()) {
+ put(entry.getKey(), entry.getValue());
+ }
+ clearStates();
}
@Override
public V put(K key, V value) {
- keyStates.put(key, State.DIRTY);
- return super.put(key, value);
+ keyStates.remove(key);
+ V old = super.put(key, value);
+ //if old value is different or null, set state to dirty
+ if (!value.equals(old)) {
+ keyStates.put(key, State.DIRTY);
+ }
+ return old;
}
@SuppressWarnings("unchecked")
@Override
public V remove(Object key) {
- if (keyStates.containsKey(key)) {
- keyStates.put((K) key, State.DELETED);
- }
- return super.remove(key);
+ keyStates.put((K) key, State.DELETED);
+ return null;
+ // We do not remove the actual entry from the map.
+ // When we keep the entries, we can compare previous state to make
Datastore
+ // puts more efficient. (In the case of new puts that are in fact
unchanged)
}
@Override
@@ -65,10 +83,18 @@ public class StatefulHashMap<K, V> exten
@Override
public void clear() {
+ // The problem with clear() is that we cannot delete entries that were not
+ // initially set on the input. This means that for a clear() to fully
+ // reflect on a datastore you have to input the full map from the store.
+ // This is acceptable for now. Another way around this is to implement
+ // some sort of "clear marker" that indicates a map should be fully
cleared,
+ // with respect to any possible new entries.
for (Entry<K, V> e : entrySet()) {
keyStates.put(e.getKey(), State.DELETED);
}
- super.clear();
+ // Do not actually clear the map, i.e. with super.clear()
+ // When we keep the entries, we can compare previous state to make
Datastore
+ // puts more efficient. (In the case of new puts that are in fact
unchanged)
}
public State getState(K key) {
@@ -95,4 +121,12 @@ public class StatefulHashMap<K, V> exten
public Map<K, State> states() {
return keyStates;
}
+
+ /* (non-Javadoc)
+ * @see org.apache.gora.persistency.StatefulMap#reuse()
+ */
+ public void reuse() {
+ super.clear();
+ clearStates();
+ }
}
Modified:
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/StatefulMap.java
URL:
http://svn.apache.org/viewvc/gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/StatefulMap.java?rev=1327922&r1=1327921&r2=1327922&view=diff
==============================================================================
---
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/StatefulMap.java
(original)
+++
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/StatefulMap.java
Thu Apr 19 12:15:35 2012
@@ -34,4 +34,10 @@ public interface StatefulMap<K, V> exten
void clearStates();
+ /**
+ * Reuse will clear the map completely with states. This is different
+ * from {@link #clear()} in that the latter only sets entries to deleted.
+ */
+ void reuse();
+
}
\ No newline at end of file
Modified:
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
URL:
http://svn.apache.org/viewvc/gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java?rev=1327922&r1=1327921&r2=1327922&view=diff
==============================================================================
---
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
(original)
+++
gora/trunk/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
Thu Apr 19 12:15:35 2012
@@ -29,6 +29,7 @@ import org.apache.gora.avro.PersistentDa
import org.apache.gora.persistency.ListGenericArray;
import org.apache.gora.persistency.Persistent;
import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.StatefulHashMap;
/**
* Base classs implementing common functionality for Persistent
@@ -99,7 +100,15 @@ public abstract class PersistentBase imp
for(int i=0; i<getFields().length; i++) {
switch(fields.get(i).schema().getType()) {
- case MAP: if(get(i) != null) ((Map)get(i)).clear(); break;
+ case MAP:
+ if(get(i) != null) {
+ if (get(i) instanceof StatefulHashMap) {
+ ((StatefulHashMap)get(i)).reuse();
+ } else {
+ ((Map)get(i)).clear();
+ }
+ }
+ break;
case ARRAY:
if(get(i) != null) {
if(get(i) instanceof ListGenericArray) {
