I noticed that mkgmap does not intern any strings. In particular, this
tile, generated by the splitter, fails to build with -Xmx3000m on
64-bit jdk under linux. With my patch, mkgmap generates the tile with
-Xmx1000m.
<bounds minlat='55.1953125' minlon='9.4921875' maxlat='56.6015625'
maxlon='11.513671875'/>
This tile has 1m nodes. Among the nodes and ways on this tile, there
are 12m tags, yet only 100k distinct tag key/value pairs; on average
each value occurs 120 times.
I explicitly do not use normal string interning because
String.intern() strings are kept forever, and I want these strings to
be GC'able after the tile is done. I trade GCability for having the
occasional string duplicated in memory by flushing the interning table
every 10k unique strings.
This code is not presently multithread safe; Ideally there should be
one string interning table for each parser/thread.
Scott
diff --git a/src/uk/me/parabola/mkgmap/reader/osm/LossyIntern.java b/src/uk/me/parabola/mkgmap/reader/osm/LossyIntern.java
new file mode 100644
index 0000000..5a6e07c
--- /dev/null
+++ b/src/uk/me/parabola/mkgmap/reader/osm/LossyIntern.java
@@ -0,0 +1,26 @@
+package uk.me.parabola.mkgmap.reader.osm;
+
+import java.util.HashMap;
+
+/** Implement an interning string pool that resets its contents every N strings */
+public class LossyIntern {
+ static private HashMap<String,String> internmap = new HashMap<String,String>();
+ static private int interncount=0;
+
+ static public String intern(String s) {
+ String out=internmap.get(s);
+ if (out != null) {
+ return out;
+ }
+ interncount++;
+ if (interncount > 100000) {
+ internmap.clear();
+ interncount = 0;
+ }
+ internmap.put(s,s);
+ return s;
+ }
+}
+
+
+
diff --git a/src/uk/me/parabola/mkgmap/reader/osm/Tags.java b/src/uk/me/parabola/mkgmap/reader/osm/Tags.java
index fd65b31..0c48318 100644
--- a/src/uk/me/parabola/mkgmap/reader/osm/Tags.java
+++ b/src/uk/me/parabola/mkgmap/reader/osm/Tags.java
@@ -65,12 +65,12 @@ public class Tags implements Iterable<String> {
Integer ind = keyPos(key);
if (ind == null)
assert false : "keyPos(" + key + ") returns null - size = " + size + ", capacity = " + capacity;
- keys[ind] = key;
+ keys[ind] = LossyIntern.intern(key);
String old = values[ind];
if (old == null)
size++;
- values[ind] = value;
+ values[ind] = LossyIntern.intern(value);
return old;
}
@@ -232,7 +232,7 @@ public class Tags implements Iterable<String> {
for(int i = 0; i < capacity; ++i) {
if(keys[i] != null && keys[i].startsWith(prefix)) {
if(removePrefix)
- map.put(keys[i].substring(prefixLen), values[i]);
+ map.put(LossyIntern.intern(keys[i].substring(prefixLen)), values[i]);
else
map.put(keys[i], values[i]);
}
_______________________________________________
mkgmap-dev mailing list
mkgmap-dev@lists.mkgmap.org.uk
http://www.mkgmap.org.uk/mailman/listinfo/mkgmap-dev
