Author: tv
Date: Sun Aug 7 09:49:54 2016
New Revision: 1755432
URL: http://svn.apache.org/viewvc?rev=1755432&view=rev
Log:
JCS-165: BlockDiskCache partial remove / group remove doesn't work
Modified:
commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java
commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestAbstract.java
commons/proper/jcs/trunk/src/changes/changes.xml
Modified:
commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java
URL:
http://svn.apache.org/viewvc/commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java?rev=1755432&r1=1755431&r2=1755432&view=diff
==============================================================================
---
commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java
(original)
+++
commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java
Sun Aug 7 09:49:54 2016
@@ -26,6 +26,8 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
@@ -40,6 +42,7 @@ import org.apache.commons.jcs.engine.beh
import org.apache.commons.jcs.engine.behavior.IElementSerializer;
import org.apache.commons.jcs.engine.behavior.IRequireScheduler;
import org.apache.commons.jcs.engine.control.group.GroupAttrName;
+import org.apache.commons.jcs.engine.control.group.GroupId;
import org.apache.commons.jcs.engine.stats.StatElement;
import org.apache.commons.jcs.engine.stats.Stats;
import org.apache.commons.jcs.engine.stats.behavior.IStatElement;
@@ -447,60 +450,17 @@ public class BlockDiskCache<K, V>
try
{
- if ( key instanceof String && key.toString().endsWith(
CacheConstants.NAME_COMPONENT_DELIMITER ) )
+ if (key instanceof String &&
key.toString().endsWith(CacheConstants.NAME_COMPONENT_DELIMITER))
{
- // remove all keys of the same name group.
- Iterator<Map.Entry<K, int[]>> iter =
this.keyStore.entrySet().iterator();
-
- while ( iter.hasNext() )
- {
- Map.Entry<K, int[]> entry = iter.next();
- K k = entry.getKey();
-
- if ( k instanceof String && k.toString().startsWith(
key.toString() ) )
- {
- int[] ded = entry.getValue();
- this.dataFile.freeBlocks( ded );
- iter.remove();
- removed = true;
- // TODO this needs to update the remove count
separately
- }
- }
+ removed = performPartialKeyRemoval((String) key);
}
- else if ( key instanceof GroupAttrName &&
((GroupAttrName<?>)key).attrName == null )
+ else if (key instanceof GroupAttrName && ((GroupAttrName<?>)
key).attrName == null)
{
- // remove all keys of the same name hierarchy.
- Iterator<Map.Entry<K, int[]>> iter =
this.keyStore.entrySet().iterator();
- while ( iter.hasNext() )
- {
- Map.Entry<K, int[]> entry = iter.next();
- K k = entry.getKey();
-
- if ( k instanceof GroupAttrName &&
-
((GroupAttrName<?>)k).groupId.equals(((GroupAttrName<?>)key).groupId))
- {
- int[] ded = entry.getValue();
- this.dataFile.freeBlocks( ded );
- iter.remove();
- removed = true;
- }
- }
+ removed = performGroupRemoval(((GroupAttrName<?>)
key).groupId);
}
else
{
- // remove single item.
- int[] ded = this.keyStore.remove( key );
- removed = ded != null;
- if ( removed )
- {
- this.dataFile.freeBlocks( ded );
- }
-
- if ( log.isDebugEnabled() )
- {
- log.debug( logCacheName + "Disk removal: Removed from key
hash, key [" + key + "] removed = "
- + removed );
- }
+ removed = performSingleKeyRemoval(key);
}
}
catch ( Exception e )
@@ -522,6 +482,102 @@ public class BlockDiskCache<K, V>
}
/**
+ * Remove all elements from the group. This does not use the iterator to
remove. It builds a
+ * list of group elements and then removes them one by one.
+ * <p>
+ * This operates under a lock obtained in doRemove().
+ * <p>
+ *
+ * @param key
+ * @return true if an element was removed
+ */
+ private boolean performGroupRemoval(GroupId key)
+ {
+ boolean removed = false;
+
+ // remove all keys of the same name group.
+ List<K> itemsToRemove = new LinkedList<K>();
+
+ // remove all keys of the same name hierarchy.
+ for (K k : keyStore.keySet())
+ {
+ if (k instanceof GroupAttrName && ((GroupAttrName<?>)
k).groupId.equals(key))
+ {
+ itemsToRemove.add(k);
+ }
+ }
+
+ // remove matches.
+ for (K fullKey : itemsToRemove)
+ {
+ // Don't add to recycle bin here
+ // https://issues.apache.org/jira/browse/JCS-67
+ performSingleKeyRemoval(fullKey);
+ removed = true;
+ // TODO this needs to update the remove count separately
+ }
+
+ return removed;
+ }
+
+ /**
+ * Iterates over the keyset. Builds a list of matches. Removes all the
keys in the list. Does
+ * not remove via the iterator, since the map impl may not support it.
+ * <p>
+ * This operates under a lock obtained in doRemove().
+ * <p>
+ *
+ * @param key
+ * @return true if there was a match
+ */
+ private boolean performPartialKeyRemoval(String key)
+ {
+ boolean removed = false;
+
+ // remove all keys of the same name hierarchy.
+ List<K> itemsToRemove = new LinkedList<K>();
+
+ for (K k : keyStore.keySet())
+ {
+ if (k instanceof String && k.toString().startsWith(key))
+ {
+ itemsToRemove.add(k);
+ }
+ }
+
+ // remove matches.
+ for (K fullKey : itemsToRemove)
+ {
+ // Don't add to recycle bin here
+ // https://issues.apache.org/jira/browse/JCS-67
+ performSingleKeyRemoval(fullKey);
+ removed = true;
+ // TODO this needs to update the remove count separately
+ }
+
+ return removed;
+ }
+
+
+ private boolean performSingleKeyRemoval(K key) {
+ boolean removed;
+ // remove single item.
+ int[] ded = this.keyStore.remove( key );
+ removed = ded != null;
+ if ( removed )
+ {
+ this.dataFile.freeBlocks( ded );
+ }
+
+ if ( log.isDebugEnabled() )
+ {
+ log.debug( logCacheName + "Disk removal: Removed from key
hash, key [" + key + "] removed = "
+ + removed );
+ }
+ return removed;
+ }
+
+ /**
* Resets the keyfile, the disk file, and the memory key map.
* <p>
* @see org.apache.commons.jcs.auxiliary.disk.AbstractDiskCache#removeAll()
Modified:
commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestAbstract.java
URL:
http://svn.apache.org/viewvc/commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestAbstract.java?rev=1755432&r1=1755431&r2=1755432&view=diff
==============================================================================
---
commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestAbstract.java
(original)
+++
commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestAbstract.java
Sun Aug 7 09:49:54 2016
@@ -20,13 +20,18 @@ package org.apache.commons.jcs.auxiliary
*/
import java.io.File;
+import java.io.IOException;
import java.io.Serializable;
import java.util.Map;
import junit.framework.TestCase;
import org.apache.commons.jcs.engine.CacheElement;
+import org.apache.commons.jcs.engine.ElementAttributes;
import org.apache.commons.jcs.engine.behavior.ICacheElement;
+import org.apache.commons.jcs.engine.behavior.IElementAttributes;
+import org.apache.commons.jcs.engine.control.group.GroupAttrName;
+import org.apache.commons.jcs.engine.control.group.GroupId;
import org.apache.commons.jcs.utils.serialization.StandardSerializer;
/** Unit tests for the Block Disk Cache */
@@ -327,8 +332,8 @@ public abstract class BlockDiskCacheUnit
oneLoadFromDisk();
}
}
-
- public void testAppendToDisk() throws Exception
+
+ public void testAppendToDisk() throws Exception
{
String cacheName = "testAppendToDisk";
BlockDiskCacheAttributes cattr = getCacheAttributes();
@@ -403,6 +408,155 @@ public abstract class BlockDiskCacheUnit
diskCache.dispose();
}
+ /**
+ * Add some items to the disk cache and then remove them one by one.
+ *
+ * @throws IOException
+ */
+ public void testRemoveItems() throws IOException
+ {
+ BlockDiskCacheAttributes cattr = getCacheAttributes();
+ cattr.setCacheName("testRemoveItems");
+ cattr.setMaxKeySize(100);
+ cattr.setDiskPath("target/test-sandbox/BlockDiskCacheUnitTest");
+ BlockDiskCache<String, String> disk = new BlockDiskCache<String,
String>(cattr);
+
+ disk.processRemoveAll();
+
+ int cnt = 25;
+ for (int i = 0; i < cnt; i++)
+ {
+ IElementAttributes eAttr = new ElementAttributes();
+ eAttr.setIsSpool(true);
+ ICacheElement<String, String> element = new CacheElement<String,
String>("testRemoveItems", "key:" + i, "data:" + i);
+ element.setElementAttributes(eAttr);
+ disk.processUpdate(element);
+ }
+
+ // remove each
+ for (int i = 0; i < cnt; i++)
+ {
+ disk.remove("key:" + i);
+ ICacheElement<String, String> element = disk.processGet("key:" +
i);
+ assertNull("Should not have received an element.", element);
+ }
+ }
+
+ /**
+ * Add some items to the disk cache and then remove them one by one.
+ * <p>
+ *
+ * @throws IOException
+ */
+ public void testRemove_PartialKey() throws IOException
+ {
+ BlockDiskCacheAttributes cattr = getCacheAttributes();
+ cattr.setCacheName("testRemove_PartialKey");
+ cattr.setMaxKeySize(100);
+ cattr.setDiskPath("target/test-sandbox/BlockDiskCacheUnitTest");
+ BlockDiskCache<String, String> disk = new BlockDiskCache<String,
String>(cattr);
+
+ disk.processRemoveAll();
+
+ int cnt = 25;
+ for (int i = 0; i < cnt; i++)
+ {
+ IElementAttributes eAttr = new ElementAttributes();
+ eAttr.setIsSpool(true);
+ ICacheElement<String, String> element = new CacheElement<String,
String>("testRemove_PartialKey", i + ":key", "data:"
+ + i);
+ element.setElementAttributes(eAttr);
+ disk.processUpdate(element);
+ }
+
+ // verify each
+ for (int i = 0; i < cnt; i++)
+ {
+ ICacheElement<String, String> element = disk.processGet(i +
":key");
+ assertNotNull("Shoulds have received an element.", element);
+ }
+
+ // remove each
+ for (int i = 0; i < cnt; i++)
+ {
+ disk.remove(i + ":");
+ ICacheElement<String, String> element = disk.processGet(i +
":key");
+ assertNull("Should not have received an element.", element);
+ }
+ }
+
+
+ /**
+ * Verify that group members are removed if we call remove with a group.
+ *
+ * @throws IOException
+ */
+ public void testRemove_Group() throws IOException
+ {
+ // SETUP
+ BlockDiskCacheAttributes cattr = getCacheAttributes();
+ cattr.setCacheName("testRemove_Group");
+ cattr.setMaxKeySize(100);
+ cattr.setDiskPath("target/test-sandbox/BlockDiskCacheUnitTest");
+ BlockDiskCache<GroupAttrName<String>, String> disk = new
BlockDiskCache<GroupAttrName<String>, String>(cattr);
+
+ disk.processRemoveAll();
+
+ String cacheName = "testRemove_Group_Region";
+ String groupName = "testRemove_Group";
+
+ int cnt = 25;
+ for (int i = 0; i < cnt; i++)
+ {
+ GroupAttrName<String> groupAttrName = getGroupAttrName(cacheName,
groupName, i + ":key");
+ CacheElement<GroupAttrName<String>, String> element = new
CacheElement<GroupAttrName<String>, String>(cacheName,
+ groupAttrName, "data:" + i);
+
+ IElementAttributes eAttr = new ElementAttributes();
+ eAttr.setIsSpool(true);
+ element.setElementAttributes(eAttr);
+
+ disk.processUpdate(element);
+ }
+
+ // verify each
+ for (int i = 0; i < cnt; i++)
+ {
+ GroupAttrName<String> groupAttrName = getGroupAttrName(cacheName,
groupName, i + ":key");
+ ICacheElement<GroupAttrName<String>, String> element =
disk.processGet(groupAttrName);
+ assertNotNull("Should have received an element.", element);
+ }
+
+ // DO WORK
+ // remove the group
+ disk.remove(getGroupAttrName(cacheName, groupName, null));
+
+ for (int i = 0; i < cnt; i++)
+ {
+ GroupAttrName<String> groupAttrName = getGroupAttrName(cacheName,
groupName, i + ":key");
+ ICacheElement<GroupAttrName<String>, String> element =
disk.processGet(groupAttrName);
+
+ // VERIFY
+ assertNull("Should not have received an element.", element);
+ }
+
+ }
+
+ /**
+ * Internal method used for group functionality.
+ * <p>
+ *
+ * @param cacheName
+ * @param group
+ * @param name
+ * @return GroupAttrName
+ */
+ private GroupAttrName<String> getGroupAttrName(String cacheName, String
group, String name)
+ {
+ GroupId gid = new GroupId(cacheName, group);
+ return new GroupAttrName<String>(gid, name);
+ }
+
/** Holder for a string and byte array. */
static class X implements Serializable
{
Modified: commons/proper/jcs/trunk/src/changes/changes.xml
URL:
http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/changes/changes.xml?rev=1755432&r1=1755431&r2=1755432&view=diff
==============================================================================
--- commons/proper/jcs/trunk/src/changes/changes.xml (original)
+++ commons/proper/jcs/trunk/src/changes/changes.xml Sun Aug 7 09:49:54 2016
@@ -20,6 +20,9 @@
</properties>
<body>
<release version="2.0" date="unreleased" description="JDK 1.6 based
major release">
+ <action issue="JCS-165" dev="tv" type="fix" due-to="Wiktor
Niesiobedzki">
+ Fix: BlockDiskCache partial remove / group remove doesn't work
+ </action>
<action issue="JCS-156" dev="sebb" type="fix" due-to="Ryan Fong">
BlockDiskCache is limited to 2GB
</action>
