Revision: 16676
http://sourceforge.net/p/gate/code/16676
Author: valyt
Date: 2013-05-03 15:13:55 +0000 (Fri, 03 May 2013)
Log Message:
-----------
First attempt at speeding up indexing of very large mention sets.
Background: when indexing annotations that have non-nominal features with very
large ranges (e.g. URIs from a large knowledge base) the indexing slows down
rapidly, presumably due to many cache misses.
Implemented solution:
- changed the caching implementation to use 3 completely independent caches for
Level 1 IDs, Level 2 IDs, and mention IDs. This allows the single Level 2 cache
to be significantly larger than each of the many Level 2 caches previously used.
- increased the default cache size for Level 2 to 10240.
- increased Level 3 (mention IDs) cache size to 1024*1024.
- added a public method that allows setting the size of the caches from inside
the index template.
Other notes:
- removed the dynamic cache re-sizing, as it turned out to be ineffectual.
- changed the helper implementation to get the ID for newly inserted rows
directly from the insert prepared statement (instead of re-running the select
statement).
- changed the way we deal with cache misses: we now supply callables that
retrieve the ID from the DB (or generate a new one by inserting a new row).
Modified Paths:
--------------
mimir/trunk/plugins/db-h2/src/gate/mimir/db/AnnotationTemplateCache.java
mimir/trunk/plugins/db-h2/src/gate/mimir/db/DBSemanticAnnotationHelper.java
Modified:
mimir/trunk/plugins/db-h2/src/gate/mimir/db/AnnotationTemplateCache.java
===================================================================
--- mimir/trunk/plugins/db-h2/src/gate/mimir/db/AnnotationTemplateCache.java
2013-05-03 09:41:57 UTC (rev 16675)
+++ mimir/trunk/plugins/db-h2/src/gate/mimir/db/AnnotationTemplateCache.java
2013-05-03 15:13:55 UTC (rev 16676)
@@ -14,7 +14,6 @@
*/
package gate.mimir.db;
-import gate.Annotation;
import gate.FeatureMap;
import gate.mimir.AbstractSemanticAnnotationHelper;
import it.unimi.dsi.fastutil.objects.Object2ShortMap;
@@ -24,51 +23,15 @@
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.Callable;
-import org.apache.log4j.Logger;
-
public class AnnotationTemplateCache {
- /**
- * Interface for the types of values returned by this cache. All Tags have a
- * long ID (if the value is -1, then the ID is not yet set). They may also
- * hold some other data used internally by the cache.
- */
- public static interface Tag {
- /**
- * Value for a Tag's ID when no ID as been set yet.
- */
- public static final long NO_ID = -1;
- /**
- * Value for a Tag's ID when no ID exists (i.e. an alternative
- * representation for when the Tag value should be null).
- */
- public static final long NULL_ID = -2;
-
- /**
- * Gets the ID associated with this tag. If the values returned is -1, then
- * the has no ID.
- *
- * @return
- */
- long getId();
-
- /**
- * Sets the ID that this tag should have. When a cache miss occurs, a new
- * tag is created and returned. Such a tag has no ID (the id value is
- * {@link #NO_ID}). Client code can then set the ID to whatever desired
- * value.
- *
- * @param newId
- */
- void setId(long newId);
- }
-
/**
* The key that goes into the level 1 cache map.
*/
- protected class NominalFeatures {
- public NominalFeatures(FeatureMap annFeats) {
+ protected class Level1Key {
+ public Level1Key(FeatureMap annFeats) {
// nominalValues is guaranteed to be non-null and to have the same size
// as owner.getNominalFeatureNames() (i.e. 0, if no nominal features)
features = new short[nominalvalues.length];
@@ -92,7 +55,7 @@
@Override
public boolean equals(Object obj) {
- return Arrays.equals(features, ((NominalFeatures)obj).features);
+ return Arrays.equals(features, ((Level1Key)obj).features);
}
private short[] features;
@@ -108,8 +71,9 @@
/**
* The type of keys that go into the level2 cache.
*/
- protected class NonNominalFeatures {
- public NonNominalFeatures(FeatureMap annFeats) {
+ protected class Level2Key {
+ public Level2Key(long level1id, FeatureMap annFeats) {
+ this.level1id = level1id;
int length = 0;
if(owner.getIntegerFeatures() != null)
length += owner.getIntegerFeatures().length;
@@ -119,7 +83,7 @@
length += owner.getTextFeatures().length;
if(owner.getUriFeatures() != null)
length += owner.getUriFeatures().length;
- values = new Object[length];
+ values = new Object[length + 1];
int i = 0;
if(owner.getIntegerFeatures() != null) {
for(String aFeature : owner.getIntegerFeatures()) {
@@ -141,12 +105,15 @@
values[i++] = annFeats.get(aFeature);
}
}
+ values[i] = level1id;
// cache the hash code.
hashcode = Arrays.hashCode(values);
}
- Object[] values;
+ private Object[] values;
+ long level1id;
+
@Override
public int hashCode() {
return hashcode;
@@ -154,66 +121,19 @@
@Override
public boolean equals(Object obj) {
- return Arrays.equals(values, ((NonNominalFeatures)obj).values);
+ return Arrays.equals(values, ((Level2Key)obj).values);
}
int hashcode;
}
/**
- * Type of values that go in the the Level1 cache
- */
- protected class Level2Cache implements Tag {
- public Level2Cache() {
- this.id = NO_ID;
- level2Cache = new LinkedHashMap<NonNominalFeatures, LongTag>() {
- @Override
- protected boolean removeEldestEntry(
- Entry<NonNominalFeatures, LongTag> eldest) {
- return size() > l2CacheSize;
- }
- };
- }
-
- long id;
-
- Map<NonNominalFeatures, LongTag> level2Cache;
-
- public long getId() {
- return id;
- }
-
- public void setId(long newId) {
- this.id = newId;
- }
- }
-
- /**
- * Type for value going in the level 2 cache (a simple wrapper for a long).
- */
- protected static class LongTag implements Tag {
- public LongTag() {
- this.id = NO_ID;
- }
-
- public long getId() {
- return id;
- }
-
- public void setId(long newId) {
- this.id = newId;
- }
-
- long id;
- }
-
- /**
* Type for keys going in the level 3 cache
*/
- protected class MentionKey {
- public MentionKey(long l1Id, long l2id, int mentionLength) {
- this.level1Id = l1Id;
- this.level2Id = l2id;
+ protected class Level3Key {
+ public Level3Key(long level1Id, long level2Id, int mentionLength) {
+ this.level1Id = level1Id;
+ this.level2Id = level2Id;
this.mentionLength = mentionLength;
this.hashcode = Arrays.hashCode(new long[]{level1Id, level2Id,
mentionLength});
}
@@ -225,7 +145,7 @@
@Override
public boolean equals(Object obj) {
- MentionKey other = (MentionKey)obj;
+ Level3Key other = (Level3Key)obj;
return other != null &&
level1Id == other.level1Id &&
level2Id == other.level2Id &&
@@ -241,18 +161,23 @@
int hashcode;
}
+ /**
+ * Value for a Tag's ID when no ID as been set yet.
+ */
+ public static final long NO_ID = -1;
+
+ /**
+ * Value for a Tag's ID when no ID exists (i.e. an alternative
+ * representation for when the Tag value should be null).
+ */
+ public static final long NULL_ID = -2;
+
private static final int DEFAULT_L1_SIZE = 512;
- private static final int DEFAULT_L2_SIZE = 64;
+ private static final int DEFAULT_L2_SIZE = 10240;
- private static Logger logger = Logger
- .getLogger(AnnotationTemplateCache.class);
+ private static final int DEFAULT_L3_SIZE = 1024 * 1024;
- /**
- * There is only one L3 cache, so we can have it quite large.
- */
- private static final int DEFAULT_L3_SIZE = 100000;
-
private static final short NULL = -1;
public AnnotationTemplateCache(AbstractSemanticAnnotationHelper owner) {
@@ -268,20 +193,35 @@
nominalvalues[i] = new Object2ShortOpenHashMap<String>();
nominalvalues[i].defaultReturnValue(NULL);
}
- level1Cache =
- new LinkedHashMap<AnnotationTemplateCache.NominalFeatures,
AnnotationTemplateCache.Level2Cache>() {
- @Override
- protected boolean removeEldestEntry(
- Entry<NominalFeatures, Level2Cache> eldest) {
- return size() > l1CacheSize;
- }
- };
- level3Cache = new LinkedHashMap<AnnotationTemplateCache.MentionKey, Tag>()
{
+ level1Cache = new LinkedHashMap<Level1Key, Long> (){
+ private static final long serialVersionUID = -7450031094311786000L;
+
@Override
- protected boolean removeEldestEntry(Entry<MentionKey, Tag> eldest) {
+ protected boolean removeEldestEntry(
+ Entry<Level1Key, Long> eldest) {
+ return size() > l1CacheSize;
+ }
+ };
+
+ level2Cache = new LinkedHashMap<Level2Key, Long> (){
+ private static final long serialVersionUID = -4387458661647300503L;
+
+ @Override
+ protected boolean removeEldestEntry(
+ Entry<Level2Key, Long> eldest) {
+ return size() > l2CacheSize;
+ }
+ };
+
+ level3Cache = new LinkedHashMap<Level3Key, Long>() {
+ private static final long serialVersionUID = -1341690439603138038L;
+
+ @Override
+ protected boolean removeEldestEntry(Entry<Level3Key, Long> eldest) {
return size() > l3CacheSize;
}
};
+
l1CacheHits = 0;
l1CacheMisses = 0;
l2CacheHits = 0;
@@ -290,9 +230,11 @@
l3CacheMisses = 0;
}
- protected Map<NominalFeatures, Level2Cache> level1Cache;
+ protected Map<Level1Key, Long> level1Cache;
+
+ protected Map<Level2Key, Long> level2Cache;
- protected Map<MentionKey, Tag> level3Cache;
+ protected Map<Level3Key, Long> level3Cache;
/**
* The helper using this cache.
@@ -323,42 +265,42 @@
private long l3CacheHits;
private long l3CacheMisses;
-
+
/**
- * Given an annotation, obtain the associated Level-1 {@link Tag}, from which
- * the ID can be retrieved. If a cache miss occurs, the returned tag will
have
- * an ID value of {@link #NO_ID} - it is the responsibility of the client
code
- * to obtain the correct ID and set it on the tag.
+ * Given an annotation, obtain the associated Level-1 ID. If a cache miss
+ * occurs, the provided callable will be called to obtain a new ID, which
will
+ * then be stored in the cache, and returned.
+ * @throws Exception if the provided callable generates an exception.
*/
- public Tag getLevel1Tag(FeatureMap annFeats) {
+ public long getLevel1Id(FeatureMap annFeats, Callable<Long> idGenerator)
throws Exception {
// build the nominal features value
- NominalFeatures nomFeats = new NominalFeatures(annFeats);
- Level2Cache l1tag = level1Cache.get(nomFeats);
- if(l1tag == null) {
+ Level1Key l1key = new Level1Key(annFeats);
+ Long l1Id = level1Cache.get(l1key);
+ if(l1Id == null) {
l1CacheMisses++;
- l1tag = new Level2Cache();
- level1Cache.put(nomFeats, l1tag);
+ l1Id = idGenerator.call();
+ level1Cache.put(l1key, l1Id);
} else {
l1CacheHits++;
}
- return l1tag;
+ return l1Id;
}
-
+
/**
- * Given an annotation and the level-1 Tag obtained previously, obtain the
- * associated level-2 {@link Tag}, from which the ID can be retrieved. If a
- * cache miss occurs, the returned tag will have an ID value of {@link
#NO_ID}
- * - it is the responsibility of the client code to obtain the correct ID and
- * set it on the tag.
+ * Given an annotation and the level-1 ID obtained previously, obtain the
+ * associated level-2 ID. If a cache miss occurs, the provided callable will
+ * be used to generate a new ID, which is then stored in the cache and
+ * returned.
+ * @throws Exception if the provided callable generates an exception.
*/
- public Tag getLevel2Tag(FeatureMap annFeats, Tag level1Tag) {
- Level2Cache l1Value = (Level2Cache)level1Tag;
- NonNominalFeatures nonNonFeats = new NonNominalFeatures(annFeats);
- LongTag level2Tag = (LongTag)l1Value.level2Cache.get(nonNonFeats);
+ public Long getLevel2Id(Long level1Tag, FeatureMap annFeats,
+ Callable<Long> idGenerator) throws Exception {
+ Level2Key nonNonFeats = new Level2Key(level1Tag, annFeats);
+ Long level2Tag = level2Cache.get(nonNonFeats);
if(level2Tag == null) {
l2CacheMisses++;
- level2Tag = new LongTag();
- l1Value.level2Cache.put(nonNonFeats, level2Tag);
+ level2Tag = idGenerator.call();
+ level2Cache.put(nonNonFeats, level2Tag);
} else {
l2CacheHits++;
}
@@ -366,21 +308,21 @@
}
/**
- * Given a Level-1 or Level-2 tag, and a mention length, obtain the Level-3
- * tag associated with the desired mention (the ID of the mention can be
- * obtained from the returned tag). If a cache miss occurs, the returned tag
- * will have an ID value of {@link #NO_ID} - it is the responsibility of the
- * client code to obtain the correct ID and set it on the tag.
+ * Given a Level-1, (optionally) a Level-2 ID, and a mention length, obtain
+ * the Level-3 ID associated with the desired mention. If a cache miss
occurs,
+ * the provided callable will be used to generate a new ID, which is then
+ * stored in the cache and returned.
+ *
+ * @throws Exception if the provided callable generates an exception.
*/
- public Tag getLevel3Tag(Tag level1tag, Tag level2tag, int length) {
- MentionKey key = new MentionKey(
- level1tag == null ? Tag.NULL_ID : level1tag.getId(),
- level2tag == null ? Tag.NULL_ID : level2tag.getId(),
- length);
- LongTag l3Tag = (LongTag)level3Cache.get(key);
+ public Long getLevel3Id(Long level1tag, Long level2tag, int length,
+ Callable<Long> idGenerator) throws Exception {
+ Level3Key key = new Level3Key(
+ level1tag, (level2tag == null ? NULL_ID : level2tag), length);
+ Long l3Tag = level3Cache.get(key);
if(l3Tag == null) {
l3CacheMisses++;
- l3Tag = new LongTag();
+ l3Tag = idGenerator.call();
level3Cache.put(key, l3Tag);
} else {
l3CacheHits++;
@@ -389,34 +331,6 @@
}
/**
- * Optimises the sizes used for level 1 and level 2 caches.
- */
- protected void adjustCacheSizes() {
- double l1hit = getL1CacheHitRatio();
- double l2hit = getL2CacheHitRatio();
- if(l1hit - l2hit > 0.2) {
- // L1 better than L2, by more than 20% -> increase L2
- if(l1CacheSize >= (level1Cache.size() * 2)) {
- // L1 at less than 50% capacity
- l1CacheSize = l1CacheSize / 2;
- l2CacheSize = l2CacheSize * 2;
- logger.info("Decreasing L1 cache size to " + l1CacheSize
- + "; Increasing L2 cache size to " + l2CacheSize + ".");
- }
- } else if(l2hit - l1hit > 0.2) {
- // L2 better than L1, by more than 20% -> increase L1
- if((l1CacheSize * 2) > level1Cache.size()) {
- l1CacheSize = l1CacheSize * 2;
- l2CacheSize = l2CacheSize / 2;
- logger.info("Increasing L1 cache size to " + l1CacheSize
- + "; Decreasing L2 cache size to " + l2CacheSize + ".");
- }
- } else {
- // nothing to adjust
- }
- }
-
- /**
* Returns the current size of the Level1 cache.
*
* @return an int value.
@@ -487,4 +401,47 @@
return (double)l3CacheHits / (l3CacheHits + l3CacheMisses);
}
}
+
+ /**
+ * @return the l1CacheSize
+ */
+ public int getL1CacheSize() {
+ return l1CacheSize;
+ }
+
+ /**
+ * @param l1CacheSize the l1CacheSize to set
+ */
+ public void setL1CacheSize(int l1CacheSize) {
+ this.l1CacheSize = l1CacheSize > 0 ? l1CacheSize : DEFAULT_L1_SIZE;
+ }
+
+ /**
+ * @return the l2CacheSize
+ */
+ public int getL2CacheSize() {
+ return l2CacheSize;
+ }
+
+ /**
+ * @param l2CacheSize the l2CacheSize to set
+ */
+ public void setL2CacheSize(int l2CacheSize) {
+ this.l2CacheSize = l2CacheSize > 0 ? l2CacheSize : DEFAULT_L2_SIZE;
+ }
+
+ /**
+ * @return the l3CacheSize
+ */
+ public int getL3CacheSize() {
+ return l3CacheSize;
+ }
+
+ /**
+ * @param l3CacheSize the l3CacheSize to set
+ */
+ public void setL3CacheSize(int l3CacheSize) {
+ this.l3CacheSize = l3CacheSize > 0 ? l3CacheSize : DEFAULT_L3_SIZE;
+ }
+
}
Modified:
mimir/trunk/plugins/db-h2/src/gate/mimir/db/DBSemanticAnnotationHelper.java
===================================================================
--- mimir/trunk/plugins/db-h2/src/gate/mimir/db/DBSemanticAnnotationHelper.java
2013-05-03 09:41:57 UTC (rev 16675)
+++ mimir/trunk/plugins/db-h2/src/gate/mimir/db/DBSemanticAnnotationHelper.java
2013-05-03 15:13:55 UTC (rev 16676)
@@ -14,7 +14,6 @@
*/
package gate.mimir.db;
-import static gate.mimir.db.AnnotationTemplateCache.Tag.NO_ID;
import gate.Annotation;
import gate.Document;
import gate.FeatureMap;
@@ -23,7 +22,6 @@
import gate.mimir.ConstraintType;
import gate.mimir.IndexConfig;
import gate.mimir.SemanticAnnotationHelper;
-import gate.mimir.db.AnnotationTemplateCache.Tag;
import gate.mimir.index.Indexer;
import gate.mimir.index.Mention;
import gate.mimir.search.QueryEngine;
@@ -43,6 +41,7 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.Callable;
import org.apache.log4j.Logger;
@@ -52,6 +51,183 @@
*/
public class DBSemanticAnnotationHelper extends
AbstractSemanticAnnotationHelper{
+ /**
+ * A callable that generates Level 1 IDs given a set of features.
+ */
+ protected class Level1IdGenerator implements Callable<Long> {
+
+ public Level1IdGenerator(FeatureMap features) {
+ this.features = features;
+ }
+
+ protected FeatureMap features;
+
+ /**
+ * Retrieves the level1 ID for the given set of features. If no ID can be
+ * found (i.e. this combination of features has not been seen before), it
+ * inserts a new row in the level 1 table, and returns the ID for it.
+ * @see java.util.concurrent.Callable#call()
+ */
+ @Override
+ public Long call() throws Exception {
+ setStatementParameters(level1SelectStmt, features);
+ ResultSet res = level1SelectStmt.executeQuery();
+ if(!res.next()) {
+ // no results found -> insert the new row
+ setStatementParameters(level1InsertStmt, features);
+ if(level1InsertStmt.executeUpdate() != 1) {
+ // the update failed
+ throw new RuntimeException("Error while inserting into database.
Annotation was lost!");
+ }
+ res = level1InsertStmt.getGeneratedKeys();
+ if(!res.next()) throw new RuntimeException(
+ "Could not insert new Level 1 row for features: " + features);
+ }
+
+ // we have found the level 1 ID
+ Long level1id = res.getLong(1);
+ // sanity check
+ if(res.next()) throw new RuntimeException(
+ "Multiple Unique IDs foud in Level 1 table for features: " +
+ features.toString());
+ return level1id;
+ }
+ }
+
+ /**
+ * A callable that generates Level 2 IDs given a set of features.
+ */
+ protected class Level2IdGenerator implements Callable<Long> {
+
+ private Long level1Id;
+
+ public Level2IdGenerator(Long level1Id, FeatureMap features) {
+ this.level1Id = level1Id;
+ this.features = features;
+ }
+
+ protected FeatureMap features;
+
+ /**
+ * Retrieves the level2 ID for the given set of features. If no ID can be
+ * found (i.e. this combination of features has not been seen before), it
+ * inserts a new row in the level 2 table, and returns the ID for it.
+ * @see java.util.concurrent.Callable#call()
+ */
+ @Override
+ public Long call() throws Exception {
+ level2SelectStmt.setLong(1, level1Id);
+ setStatementParameters(level2SelectStmt, features);
+ ResultSet res = level2SelectStmt.executeQuery();
+ if(!res.next()) {
+ // no results -> insert new row
+ level2InsertStmt.setLong(1, level1Id);
+ setStatementParameters(level2InsertStmt, features);
+ if(level2InsertStmt.executeUpdate() != 1) {
+ // the update failed
+ throw new RuntimeException(
+ "Could not insert new Level 2 row for Level 1 ID: \"" + level1Id +
+ "\" and features: " + features);
+ }
+ res = level2InsertStmt.getGeneratedKeys();
+ if(!res.next()) throw new RuntimeException(
+ "Could not insert new Level 2 row for Level 1 ID: \"" + level1Id +
+ "\" and features: " + features);
+ }
+
+ // we have found the level 2 ID
+ Long level2Id = res.getLong(1);
+ // sanity check
+ if(res.next()) {
+ throw new RuntimeException(
+ "Multiple Unique IDs found in Level 2 table for Level 1 ID: \"" +
+ level1Id + "\" and features: " + features);
+ }
+ return level2Id;
+ }
+ }
+
+ /**
+ * A callable that generates Level 3 IDs (i.e. mention IDs) given a Level 1
+ * ID and/or a Level 2 ID, and a mention length.
+ */
+ protected class Level3IdGenerator implements Callable<Long> {
+
+ private Long level1Id;
+
+ private Long level2Id;
+
+ private int mentionLength;
+
+ public Level3IdGenerator(Long level1Id, Long level2Id, int mentionLength) {
+ super();
+ this.level1Id = level1Id;
+ this.level2Id = level2Id;
+ this.mentionLength = mentionLength;
+ }
+
+
+ /* (non-Javadoc)
+ * @see java.util.concurrent.Callable#call()
+ */
+ @Override
+ public Long call() throws Exception {
+ mentionsSelectStmt.setLong(1, level1Id);
+ if(level2Used) {
+ if(level2Id != null) {
+ mentionsSelectStmt.setLong(2, level1Id);
+ } else {
+ mentionsSelectStmt.setNull(2, Types.BIGINT);
+ }
+ mentionsSelectStmt.setInt(3, mentionLength);
+ } else {
+ mentionsSelectStmt.setInt(2, mentionLength);
+ }
+
+ ResultSet res = mentionsSelectStmt.executeQuery();
+ if(!res.next()) {
+ // no results -> insert new row
+ mentionsInsertStmt.setLong(1, level1Id);
+ if(level2Used) {
+ if(level2Id != null) {
+ mentionsInsertStmt.setLong(2, level2Id);
+ } else {
+ mentionsInsertStmt.setNull(2, Types.BIGINT);
+ }
+ mentionsInsertStmt.setInt(3, mentionLength);
+ } else {
+ mentionsInsertStmt.setInt(2, mentionLength);
+ }
+
+ if(mentionsInsertStmt.executeUpdate() != 1) {
+ // the update failed
+ throw new RuntimeException(
+ "Could not insert new mention ID for Level 1 ID: " + level1Id +
+ ", Level 2 ID: " + level2Id + ", and mention length: " +
+ mentionLength);
+ }
+ res = mentionsInsertStmt.getGeneratedKeys();
+ if(!res.next()) {
+ throw new RuntimeException(
+ "Could not insert new mention ID for Level 1 ID: " + level1Id +
+ ", Level 2 ID: " + level2Id + ", and mention length: " +
+ mentionLength);
+ }
+ }
+
+ // we have found the level 3 (mention) ID
+ Long mentionId = res.getLong(1);
+ // sanity check
+ if(res.next()){
+ throw new RuntimeException(
+ "Multiple Unique IDs foud in mentions table for Level 1 ID: " +
+ level1Id + ", Level 2 ID: " + level2Id +
+ ", and mention length: " + mentionLength);
+ }
+ return mentionId;
+ }
+ }
+
private static final long serialVersionUID = 2734946594117068194L;
/**
@@ -574,133 +750,34 @@
try {
// find the level 1 ID
- Tag level1Tag = cache.getLevel1Tag(featuresToIndex);
- while(level1Tag.getId() == NO_ID){
- setStatementParameters(level1SelectStmt, featuresToIndex);
- ResultSet res = level1SelectStmt.executeQuery();
- if(res.next()) {
- // we have found the level 1 ID
- level1Tag.setId(res.getLong(1));
- // sanity check
- if(res.next()) throw new RuntimeException(
- "Multiple Unique IDs foud in level 1 table for annotation "
+
- ann.toString());
- } else {
- // insert the new row
- setStatementParameters(level1InsertStmt, featuresToIndex);
- if(level1InsertStmt.executeUpdate() != 1) {
- // the update failed
- logger.error("Error while inserting into database. Annotation was
lost!");
- return new String[]{};
- }
- dbConnection.commit();
- }
- }
+ Long level1Tag = cache.getLevel1Id(featuresToIndex,
+ new Level1IdGenerator(featuresToIndex));
// find the Level-1 Mention ID (ignoring the L2 values)
- Tag mentionL1Tag = cache.getLevel3Tag(level1Tag, null, length);
- Tag mentionL2Tag = null;
- while(mentionL1Tag.getId() == NO_ID){
- mentionsSelectStmt.setLong(1, level1Tag.getId());
- if(level2Used) {
- mentionsSelectStmt.setNull(2, Types.BIGINT);
- mentionsSelectStmt.setInt(3,length);
- } else {
- mentionsSelectStmt.setInt(2,length);
- }
- ResultSet res = mentionsSelectStmt.executeQuery();
- if(res.next()) {
- // we have found the level 2 ID
- mentionL1Tag.setId(res.getLong(1));
- // sanity check
- if(res.next()) throw new RuntimeException(
- "Multiple Unique IDs foud in mentions table for annotation "
+
- "(of length "+ length + "):\n" +ann.toString());
- } else {
- // insert the new row
- mentionsInsertStmt.setLong(1, level1Tag.getId());
- if(level2Used) {
- mentionsInsertStmt.setNull(2, Types.BIGINT);
- mentionsInsertStmt.setInt(3,length);
- } else {
- mentionsInsertStmt.setInt(2,length);
- }
- if(mentionsInsertStmt.executeUpdate() != 1) {
- // the update failed
- logger.error("Error while inserting into database. Annotation was
lost!");
- return new String[]{};
- }
- dbConnection.commit();
- }
- }
+ Long mentionL1Tag = cache.getLevel3Id(level1Tag, null, length,
+ new Level3IdGenerator(level1Tag, null, length));
+ Long mentionL2Tag = null;
if(level2Used){
// find the level 2 ID
- Tag level2Tag = cache.getLevel2Tag(featuresToIndex, level1Tag);
- while(level2Tag.getId() == NO_ID){
- level2SelectStmt.setLong(1, level1Tag.getId());
- setStatementParameters(level2SelectStmt, featuresToIndex);
-
- ResultSet res = level2SelectStmt.executeQuery();
- if(res.next()) {
- // we have found the level 2 ID
- level2Tag.setId(res.getLong(1));
- // sanity check
- if(res.next()) throw new RuntimeException(
- "Multiple Unique IDs found in level 2 table for annotation
" +
- ann.toString());
- } else {
- // insert the new row
- level2InsertStmt.setLong(1, level1Tag.getId());
- setStatementParameters(level2InsertStmt, featuresToIndex);
- if(level2InsertStmt.executeUpdate() != 1) {
- // the update failed
- logger.error("Error while inserting into database. Annotation
was lost!");
- return new String[]{};
- }
- dbConnection.commit();
- }
- }
+ Long level2Tag = cache.getLevel2Id(level1Tag, featuresToIndex,
+ new Level2IdGenerator(level1Tag, featuresToIndex));
+
// find the Level-2 Mention ID
- mentionL2Tag = cache.getLevel3Tag(level1Tag, level2Tag, length);
- while(mentionL2Tag.getId() == NO_ID){
- mentionsSelectStmt.setLong(1,level1Tag.getId());
- mentionsSelectStmt.setLong(2, level2Tag.getId());
- mentionsSelectStmt.setInt(3,length);
-
- ResultSet res = mentionsSelectStmt.executeQuery();
- if(res.next()) {
- // we have found the level 2 ID
- mentionL2Tag.setId(res.getLong(1));
- // sanity check
- if(res.next()) throw new RuntimeException(
- "Multiple Unique IDs foud in mentions table for annotation
" +
- "(of length "+ length + "):\n" +ann.toString());
- } else {
- // insert the new row
- mentionsInsertStmt.setLong(1, level1Tag.getId());
- mentionsInsertStmt.setLong(2, level2Tag.getId());
- mentionsInsertStmt.setInt(3,length);
- if(mentionsInsertStmt.executeUpdate() != 1) {
- // the update failed
- logger.error("Error while inserting into database. Annotation
was lost!");
- return new String[]{};
- }
- dbConnection.commit();
- }
- }
+ mentionL2Tag = cache.getLevel3Id(level1Tag, level2Tag, length,
+ new Level3IdGenerator(level1Tag, level2Tag, length));
}
// now we finally have the mention ID
if(level2Used) {
return new String[] {
- annotationType + ":" + mentionL1Tag.getId(),
- annotationType + ":" + mentionL2Tag.getId()};
+ annotationType + ":" + mentionL1Tag,
+ annotationType + ":" + mentionL2Tag};
} else {
return new String[] {
- annotationType + ":" + mentionL1Tag.getId()};
+ annotationType + ":" + mentionL1Tag};
}
- } catch(SQLException e) {
+ } catch(Exception e) {
// something went bad: we can't fix it :(
logger.error("Error while interogating database. Annotation was lost!",
e);
return new String[]{};
@@ -1168,10 +1245,6 @@
+ ", "
+ (Double.isNaN(l3ratio) ? "N/A" :
percentFormat.format(l3ratio)));
docsSoFar++;
- if(docsSoFar % 200 == 0) {
- // every 200 docs, adjust the cache sizes
- cache.adjustCacheSizes();
- }
} else {
logger.debug("Cache size(" + annotationType + "): null");
}
@@ -1221,4 +1294,26 @@
return null;
}
+
+ /**
+ * Sets the size for the three level caches used by this helper.
+ *
+ * A negative value for each cache size sets the cache size to its default
+ * value.
+ *
+ * @param level1 the size for the Level 1 cache. The Level 1 cache stores
+ * previously seen combinations of nominal feature values.
+ *
+ * @param level2 the size for the Level 2 cache. The Level 2 cache stores
+ * previously seen combinations of non-nominal feature values.
+ *
+ * @param level3 the size for the Level 3 cache. The Level 1 cache stores
+ * previously seen mention IDs.
+ */
+ public void setCacheSizes(int level1, int level2, int level3) {
+ cache.setL1CacheSize(level1);
+ cache.setL2CacheSize(level2);
+ cache.setL3CacheSize(level3);
+ }
+
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Get 100% visibility into Java/.NET code with AppDynamics Lite
It's a free troubleshooting tool designed for production
Get down to code-level detail for bottlenecks, with <2% overhead.
Download for free and get started troubleshooting in minutes.
http://p.sf.net/sfu/appdyn_d2d_ap2
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
