1 GB of heap size could not compile 62 MB osm file
time java -Xmx1012m -jar
/home/maning/osm/routable_garmin/mkgmap/trunk/dist/mkgmap.jar
--code-page=1252 --tdbfile --latin1 --country-abbr=PHI
--country-name=PHILIPPINES --remove-short-arcs --route
--road-name-pois --add-pois-to-areas --location-autofill=2
--family-id=639 --family-name="OSM_PHIL" --overview-mapname=40000001
--series-name="OSM_PHIL" --description="OSM_PHIL" --generate-sea
/home/maning/osm/routable_garmin/magellan/20090713/manila_20090713.osm
generating sea
merging: 33 4263797 33615705
merging: 32 4591652 4591687
merging: 31 4591978 33658311
merging: 30 4592000 4611686018427387909
merging: 29 4592029 4611686018427387908
merging: 28 4598416 4598418
merging: 27 23225661 23225148
merging: 26 23959594 33050431
merging: 25 27428886 32978387
merging: 24 27635420 28340210
merging: 23 28340457 4611686018427387916
merging: 22 28538727 28538728
merging: 21 31263118 4611686018427387918
merging: 20 31390236 31390245
merging: 19 33615706 33615707
merging: 18 33615793 33658198
merging: 17 33658197 4611686018427387907
merging: 16 33677950 4611686018427387910
merging: 15 4611686018427387912 4611686018427387912
merging: 15 4611686018427387914 4611686018427387913
merging: 14 4611686018427387915 4611686018427387926
merging: 13 4611686018427387917 4611686018427387927
merging: 12 4611686018427387919 4611686018427387928
merging: 11 4611686018427387920 4611686018427387929
merging: 10 4611686018427387921 33615708
merging: 9 4611686018427387922 4611686018427387923
merging: 8 4611686018427387925 4611686018427387925
merging: 8 4611686018427387932 4611686018427387931
merging: 7 4611686018427387933 4611686018427387933
merging: 7 4611686018427387935 4611686018427387935
merging: 7 4611686018427387936 4611686018427387936
merging: 7 4611686018427387937 4611686018427387937
merging: 7 4611686018427387938 4611686018427387938
merging: 7 4611686018427387939 4611686018427387939
merging: 7 4611686018427387940 4611686018427387940
merging: 7 4611686018427387941 4611686018427387941
merging: 7 4611686018427387942 4611686018427387942
merging: 7 4611686018427387943 4611686018427387943
merging: 7 4611686018427387944 4611686018427387944
merging: 7 4611686018427387945 4611686018427387945
merging: 7 4611686018427387946 4611686018427387946
merging: 7 4611686018427387947 4611686018427387947
merging: 7 4611686018427387948 4611686018427387948
merging: 7 4611686018427387949 4611686018427387949
merging: 7 4611686018427387950 4611686018427387950
merging: 7 4611686018427387951 4611686018427387951
merging: 7 4611686018427387952 4611686018427387952
^[[BSEVERE (Main): java.util.concurrent.ExecutionException:
java.lang.OutOfMemoryError: Java heap space
java.util.concurrent.ExecutionException: java.lang.OutOfMemoryError:
Java heap space
at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:252)
at java.util.concurrent.FutureTask.get(FutureTask.java:111)
at uk.me.parabola.mkgmap.main.Main.endOptions(Main.java:289)
at
uk.me.parabola.mkgmap.CommandArgsReader.readArgs(CommandArgsReader.java:124)
at uk.me.parabola.mkgmap.main.Main.main(Main.java:100)
Caused by: java.lang.OutOfMemoryError: Java heap space
at java.util.Arrays.copyOf(Arrays.java:2772)
at java.util.Arrays.copyOf(Arrays.java:2746)
at java.util.ArrayList.ensureCapacity(ArrayList.java:187)
at java.util.ArrayList.addAll(ArrayList.java:499)
at
uk.me.parabola.mkgmap.reader.osm.xml.Osm5XmlHandler.concatenateWays(Osm5XmlHandler.java:838)
at
uk.me.parabola.mkgmap.reader.osm.xml.Osm5XmlHandler.generateSeaPolygon(Osm5XmlHandler.java:793)
at
uk.me.parabola.mkgmap.reader.osm.xml.Osm5XmlHandler.endDocument(Osm5XmlHandler.java:457)
at
com.sun.org.apache.xerces.internal.parsers.AbstractSAXParser.endDocument(AbstractSAXParser.java:740)
at
com.sun.org.apache.xerces.internal.xinclude.XIncludeHandler.endDocument(XIncludeHandler.java:1122)
at
com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl.scanDocument(XMLDocumentFragmentScannerImpl.java:491)
at
com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:810)
at
com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:740)
at
com.sun.org.apache.xerces.internal.parsers.XMLParser.parse(XMLParser.java:110)
at
com.sun.org.apache.xerces.internal.parsers.AbstractSAXParser.parse(AbstractSAXParser.java:1208)
at
com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl$JAXPSAXParser.parse(SAXParserImpl.java:525)
at javax.xml.parsers.SAXParser.parse(SAXParser.java:392)
at javax.xml.parsers.SAXParser.parse(SAXParser.java:195)
at
uk.me.parabola.mkgmap.reader.osm.xml.Osm5MapDataSource.load(Osm5MapDataSource.java:80)
at uk.me.parabola.mkgmap.main.MapMaker.loadFromFile(MapMaker.java:148)
at uk.me.parabola.mkgmap.main.MapMaker.makeMap(MapMaker.java:56)
at uk.me.parabola.mkgmap.main.Main$1.call(Main.java:168)
at uk.me.parabola.mkgmap.main.Main$1.call(Main.java:166)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Exiting - if you want to carry on regardless, use the --keep-going option
On Sun, Aug 2, 2009 at 3:56 PM, Dermot McNally<[email protected]> wrote:
> This exhausted the 2G of heap space I had allocated when I tried it on
> a map of Ireland. Are there known practical limits I should try to
> stay within?
>
> Dermot
>
> 2009/8/1 Christian Gawron <[email protected]>:
>> Hi,
>>
>> the attached patch adds a sea polygon (based on the bounding box of the
>> map) and a multipolygon relation with all lines tagged as natural=coastline
>> as inner elements. The code merges coastline components as far as possible.
>> The patch also contains the multipolygon patch by Rudi which wasn't commited
>> so far (without this patch, the rendering fails).
>>
>> The sea polygon is created as "natural=sea", for which I added a mapping to
>> garmin type 0x32.
>>
>> Caveat: I have only tested this for islands (Corsica) so far.
>>
>> Best wishes
>> Christian
>>
>> Index: src/uk/me/parabola/mkgmap/reader/osm/Way.java
>> ===================================================================
>> --- src/uk/me/parabola/mkgmap/reader/osm/Way.java (Revision 1115)
>> +++ src/uk/me/parabola/mkgmap/reader/osm/Way.java (Arbeitskopie)
>> @@ -76,6 +76,10 @@
>> }
>> }
>>
>> + public boolean isClosed() {
>> + return points.get(0).equals(points.get(points.size()-1));
>> + }
>> +
>> /**
>> * A simple representation of this way.
>> * @return A string with the name and start point
>> Index: src/uk/me/parabola/mkgmap/reader/osm/MultiPolygonRelation.java
>> ===================================================================
>> --- src/uk/me/parabola/mkgmap/reader/osm/MultiPolygonRelation.java
>> (Revision 1115)
>> +++ src/uk/me/parabola/mkgmap/reader/osm/MultiPolygonRelation.java
>> (Arbeitskopie)
>> @@ -5,7 +5,6 @@
>> import java.util.List;
>> import java.util.Map;
>>
>> -import uk.me.parabola.imgfmt.Utils;
>> import uk.me.parabola.imgfmt.app.Coord;
>>
>> /**
>> @@ -23,8 +22,9 @@
>> * this because the type of the relation is not known until after all
>> * its tags are read in.
>> * @param other The relation to base this one on.
>> + * @param wayMap Map of all ways.
>> */
>> - public MultiPolygonRelation(Relation other) {
>> + public MultiPolygonRelation(Relation other, Map<Long, Way> wayMap) {
>> setId(other.getId());
>> for (Map.Entry<Element, String> pairs:
>> other.getRoles().entrySet()){
>> addElement(pairs.getValue(), pairs.getKey());
>> @@ -33,8 +33,16 @@
>>
>> if (value != null && pairs.getKey() instanceof Way) {
>> Way way = (Way) pairs.getKey();
>> - if (value.equals("outer"))
>> - outer = way;
>> + if (value.equals("outer")){
>> + // duplicate outer way and remove
>> tags for cascaded multipolygons
>> + outer = new Way(-way.getId());
>> + outer.copyTags(way);
>> + for(Coord point: way.getPoints())
>> + outer.addPoint(point);
>> + wayMap.put(outer.getId(), outer);
>> + if (way.getTags() != null)
>> + way.getTags().removeAll();
>> + }
>> else if (value.equals("inner"))
>> inners.add(way);
>> }
>> @@ -52,11 +60,20 @@
>> {
>> for (Way w: inners) {
>> if (w != null) {
>> - List<Coord> pts = w.getPoints();
>> - int[] insert =
>> findCpa(outer.getPoints(), pts);
>> - if (insert[0] > 0)
>> - insertPoints(pts, insert[0],
>> insert[1]);
>> - pts.clear();
>> + int[] insert =
>> findCpa(outer.getPoints(), w.getPoints());
>> + //if (insert[0] > 0)
>> + insertPoints(w, insert[0],
>> insert[1]);
>> +
>> + // remove tags from inner way that
>> are available in the outer way
>> + if (outer.getTags() != null){
>> + for (Map.Entry<String,
>> String> mapTags: outer.getTags().getKeyValues().entrySet()){
>> + String key =
>> mapTags.getKey();
>> + String value =
>> mapTags.getValue();
>> + if (w.getTag(key) !=
>> null)
>> + if
>> (w.getTag(key).equals(value))
>> +
>> w.deleteTag(key);
>> + }
>> + }
>> }
>> }
>> }
>> @@ -64,22 +81,39 @@
>>
>> /**
>> * Insert Coordinates into the outer way.
>> - * @param inList List of Coordinates to be inserted
>> + * @param way Way to be inserted
>> * @param out Coordinates will be inserted after this point in the
>> outer way.
>> * @param in Points will be inserted starting at this index,
>> * then from element 0 to (including) this element;
>> */
>> - private void insertPoints(List<Coord> inList, int out, int in){
>> + private void insertPoints(Way way, int out, int in){
>> List<Coord> outList = outer.getPoints();
>> + List<Coord> inList = way.getPoints();
>> int index = out+1;
>> for (int i = in; i < inList.size(); i++)
>> outList.add(index++, inList.get(i));
>> - for (int i = 0; i <= in; i++)
>> + for (int i = 0; i < in; i++)
>> outList.add(index++, inList.get(i));
>> -
>> - //with this line commented we get triangles, when
>> uncommented some areas disappear
>> - // at least in mapsource, on device itself looks OK.
>> - outList.add(index,outList.get(out));
>> +
>> + if (outer.getPoints().size() < 32){
>> + outList.add(index++, inList.get(in));
>> + outList.add(index, outList.get(out));
>> + }
>> + else {
>> + // we shift the nodes to avoid duplicate nodes
>> (large areas only)
>> + int oLat = outList.get(out).getLatitude();
>> + int oLon = outList.get(out).getLongitude();
>> + int iLat = inList.get(in).getLatitude();
>> + int iLon = inList.get(in).getLongitude();
>> + if ((oLat - iLat) > (oLon - iLon)){
>> + outList.add(index++, new Coord(iLat-1,
>> iLon));
>> + outList.add(index, new Coord(oLat-1, oLon));
>> + }
>> + else{
>> + outList.add(index++, new Coord(iLat,
>> iLon-1));
>> + outList.add(index, new Coord(oLat, oLon-1));
>> + }
>> + }
>> }
>>
>> /**
>> Index: src/uk/me/parabola/mkgmap/reader/osm/xml/Osm5XmlHandler.java
>> ===================================================================
>> --- src/uk/me/parabola/mkgmap/reader/osm/xml/Osm5XmlHandler.java
>> (Revision 1115)
>> +++ src/uk/me/parabola/mkgmap/reader/osm/xml/Osm5XmlHandler.java
>> (Arbeitskopie)
>> @@ -19,6 +19,7 @@
>> import java.util.ArrayList;
>> import java.util.HashMap;
>> import java.util.IdentityHashMap;
>> +import java.util.Iterator;
>> import java.util.LinkedHashMap;
>> import java.util.List;
>> import java.util.Map;
>> @@ -65,6 +66,7 @@
>> private final Map<String, Long> fakeIdMap = new HashMap<String,
>> Long>();
>> private final List<Node> exits = new ArrayList<Node>();
>> private final List<Way> motorways = new ArrayList<Way>();
>> + private final List<Way> shoreline = new ArrayList<Way>();
>>
>> private static final int MODE_NODE = 1;
>> private static final int MODE_WAY = 2;
>> @@ -92,6 +94,7 @@
>> private final boolean ignoreTurnRestrictions;
>> private final boolean linkPOIsToWays;
>> private final boolean routing;
>> + private final boolean generateSea;
>> private final Double minimumArcLength;
>> private final String frigRoundabouts;
>>
>> @@ -105,6 +108,7 @@
>> }
>> linkPOIsToWays = props.getProperty("link-pois-to-ways",
>> false);
>> ignoreBounds = props.getProperty("ignore-osm-bounds", false);
>> + generateSea = props.getProperty("generate-sea", false);
>> routing = props.containsKey("route");
>> String rsa = props.getProperty("remove-short-arcs", null);
>> if(rsa != null)
>> @@ -370,6 +374,8 @@
>> if("motorway".equals(highway) ||
>> "trunk".equals(highway))
>> motorways.add(currentWay);
>> + if(generateSea &&
>> "coastline".equals(currentWay.getTag("natural")))
>> + shoreline.add(currentWay);
>> currentWay = null;
>> // ways are processed at the end of the
>> document,
>> // may be changed by a Relation class
>> @@ -399,7 +405,7 @@
>> String type = currentRelation.getTag("type");
>> if (type != null) {
>> if ("multipolygon".equals(type))
>> - currentRelation = new
>> MultiPolygonRelation(currentRelation);
>> + currentRelation = new
>> MultiPolygonRelation(currentRelation, wayMap);
>> else if("restriction".equals(type)) {
>>
>> if(ignoreTurnRestrictions)
>> @@ -446,6 +452,10 @@
>> }
>>
>> coordMap = null;
>> +
>> + if (generateSea)
>> + generateSeaPolygon(shoreline);
>> +
>> for (Relation r : relationMap.values())
>> converter.convertRelation(r);
>>
>> @@ -746,4 +756,95 @@
>> return fakeIdVal;
>> }
>> }
>> +
>> + private void generateSeaPolygon(List<Way> shoreline) {
>> + System.out.printf("generating sea\n");
>> + long seaId;
>> + seaId = (1L << 62) + nextFakeId++;
>> + Way sea = new Way(seaId);
>> + wayMap.put(seaId, sea);
>> + Area bbox = mapper.getBounds();
>> + Coord nw = new Coord(bbox.getMinLat(), bbox.getMinLong());
>> + Coord ne = new Coord(bbox.getMinLat(), bbox.getMaxLong());
>> + Coord sw = new Coord(bbox.getMaxLat(), bbox.getMinLong());
>> + Coord se = new Coord(bbox.getMaxLat(), bbox.getMaxLong());
>> + sea.addPoint(nw);
>> + sea.addPoint(ne);
>> + sea.addPoint(se);
>> + sea.addPoint(sw);
>> + sea.addPoint(nw);
>> + sea.addTag("natural", "sea");
>> +
>> + Relation seaRelation = new GeneralRelation((1L << 62) +
>> nextFakeId++);
>> + seaRelation.addTag("type", "multipolygon");
>> + seaRelation.addElement("outer", sea);
>> +
>> + List<Way> islands = new ArrayList();
>> +
>> + // handle islands (closes shoreline components) first (they're
>> easy)
>> + Iterator<Way> it = shoreline.iterator();
>> + while (it.hasNext()) {
>> + Way w = it.next();
>> + if (w.isClosed()) {
>> + islands.add(w);
>> + it.remove();
>> + }
>> + }
>> + concatenateWays(shoreline);
>> + // there may be more islands now
>> + it = shoreline.iterator();
>> + while (it.hasNext()) {
>> + Way w = it.next();
>> + if (w.isClosed()) {
>> + System.out.println("island after concatenating\n");
>> + islands.add(w);
>> + it.remove();
>> + }
>> + }
>> +
>> + for (Way w : islands) {
>> + seaRelation.addElement("inner", w);
>> + }
>> + for (Way w : shoreline) {
>> + seaRelation.addElement("inner", w);
>> + }
>> + seaRelation = new MultiPolygonRelation(seaRelation, wayMap);
>> + relationMap.put(seaId, seaRelation);
>> + seaRelation.processElements();
>> + }
>> +
>> + public void concatenateWays(List<Way> ways) {
>> + Map<Coord, Way> beginMap = new HashMap();
>> +
>> + for (Way w : ways) {
>> + if (!w.isClosed()) {
>> + List<Coord> points = w.getPoints();
>> + beginMap.put(points.get(0), w);
>> + }
>> + }
>> +
>> + int merged = 1;
>> + while (merged > 0) {
>> + merged = 0;
>> + for (Way w1 : ways) {
>> + List<Coord> points1 = w1.getPoints();
>> + Way w2 = beginMap.get(points1.get(points1.size()-1));
>> +
>> + if (w2 != null) {
>> + System.out.printf("merging: %d %d %d\n",
>> ways.size(), w1.getId(), w2.getId());
>> + List<Coord> points2 = w2.getPoints();
>> + Way wm = new Way((1L << 62) + nextFakeId++);
>> + wm.getPoints().addAll(points1);
>> + wm.getPoints().addAll(points2);
>> + ways.remove(w1);
>> + ways.remove(w2);
>> + beginMap.remove(points2.get(0));
>> + ways.add(wm);
>> + beginMap.put(points1.get(0), wm);
>> + merged++;
>> + break;
>> + }
>> + }
>> + }
>> + }
>> }
>> Index: src/uk/me/parabola/mkgmap/reader/osm/Element.java
>> ===================================================================
>> --- src/uk/me/parabola/mkgmap/reader/osm/Element.java (Revision 1115)
>> +++ src/uk/me/parabola/mkgmap/reader/osm/Element.java (Arbeitskopie)
>> @@ -86,6 +86,7 @@
>> * element.
>> */
>> public void copyTags(Element other) {
>> + if (other.tags != null)
>> tags = other.tags.copy();
>> }
>>
>> @@ -97,4 +98,8 @@
>> if (this.name == null)
>> this.name = name;
>> }
>> +
>> + public Tags getTags() {
>> + return tags;
>> + }
>> }
>> Index: src/uk/me/parabola/mkgmap/reader/osm/Tags.java
>> ===================================================================
>> --- src/uk/me/parabola/mkgmap/reader/osm/Tags.java (Revision 1115)
>> +++ src/uk/me/parabola/mkgmap/reader/osm/Tags.java (Arbeitskopie)
>> @@ -16,7 +16,9 @@
>> */
>> package uk.me.parabola.mkgmap.reader.osm;
>>
>> +import java.util.HashMap;
>> import java.util.Iterator;
>> +import java.util.Map;
>>
>> /**
>> * Store the tags that belong to an Element.
>> @@ -111,7 +113,7 @@
>> }
>> return null;
>> }
>> -
>> +
>> /**
>> * Make a deep copy of this object.
>> * @return A copy of this object.
>> @@ -262,4 +264,22 @@
>> put(e.key, e.value);
>> }
>> }
>> -}
>> +
>> + public void removeAll() {
>> + for (int i = 0; i < capacity; i++){
>> + keys[i] = null;
>> + values[i] = null;
>> + }
>> + size = 0;
>> + }
>> +
>> + public Map<String, String> getKeyValues() {
>> + Map<String, String> tagMap = new HashMap<String, String>();
>> + for (int i = 0; i < capacity; i++)
>> + if (keys[i] != null && values[i] != null)
>> + tagMap.put(keys[i], values[i]);
>> + return tagMap;
>> + }
>> +
>> +
>> +}
>> \ No newline at end of file
>> Index: resources/styles/default/polygons
>> ===================================================================
>> --- resources/styles/default/polygons (Revision 1115)
>> +++ resources/styles/default/polygons (Arbeitskopie)
>> @@ -55,6 +55,7 @@
>> natural=mud [0x51 resolution 20]
>> natural=scrub [0x4f resolution 20]
>> natural=water [0x3c resolution 20]
>> +natural=sea [0x32 resolution 10]
>> natural=wood [0x50 resolution 18]
>>
>> place=village [0x03 resolution 18]
>>
>> _______________________________________________
>> mkgmap-dev mailing list
>> [email protected]
>> http://www.mkgmap.org.uk/mailman/listinfo/mkgmap-dev
>>
>
>
>
> --
> --------------------------------------
> Iren sind menschlich
> _______________________________________________
> mkgmap-dev mailing list
> [email protected]
> http://www.mkgmap.org.uk/mailman/listinfo/mkgmap-dev
>
--
cheers,
maning
------------------------------------------------------
"Freedom is still the most radical idea of all" -N.Branden
wiki: http://esambale.wikispaces.com/
blog: http://epsg4253.wordpress.com/
------------------------------------------------------
_______________________________________________
mkgmap-dev mailing list
[email protected]
http://www.mkgmap.org.uk/mailman/listinfo/mkgmap-dev
