Index: src/uk/me/parabola/imgfmt/app/mdr/MDRFile.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/MDRFile.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/MDRFile.java (working copy) @@ -296,6 +296,8 @@ mdr18.setPoiTypes(mdr19.getPoiTypes()); mdr19.release(); writeSection(writer, 18, mdr18); + } else { + mdr19.release(); } writeSection(writer, 10, mdr10); Index: src/uk/me/parabola/imgfmt/app/mdr/Mdr7.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/Mdr7.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/Mdr7.java (working copy) @@ -14,7 +14,9 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import uk.me.parabola.imgfmt.MapFailedException; import uk.me.parabola.imgfmt.app.ImgFileWriter; @@ -77,6 +79,7 @@ if (prefix >= MAX_NAME_OFFSET || suffix >= MAX_NAME_OFFSET) return; + name = name.intern(); Mdr7Record st = new Mdr7Record(); st.setMapIndex(mapId); st.setLabelOffset(lblOffset); @@ -168,13 +171,11 @@ */ protected void preWriteImpl() { Sort sort = getConfig().getSort(); - List> sortedStreets = new ArrayList<>(allStreets.size()); - for (Mdr7Record m : allStreets) { - sortedStreets.add(new MultiSortKey<>( - sort.createSortKey(m, m.getPartialName()), - sort.createSortKey(m, m.getInitialPart(), m.getMapIndex()), - null)); - } + List> sortedStreets; + if (splitName) + sortedStreets = createMultiSortKeys (sort); + else + sortedStreets = createSimpleSortKeys (sort); Collections.sort(sortedStreets); // De-duplicate the street names so that there is only one entry @@ -200,8 +201,86 @@ // release memory sortedStreets.set(i, null); } + return; } + /** + * Create sort keys for each entry in allStreets. Some keys appear only once, others appear very often. + * We only use a cache for those which appear often enough. + * @param sort the Sort instance + * @return list of keys + */ + private List> createMultiSortKeys(Sort sort) { + List> streetKeys = new ArrayList<>(allStreets.size()); + // step1 : find out how often each key appears + HashMap countMap1 = new HashMap<>(); + HashMap countMap2 = new HashMap<>(); + for (Mdr7Record m : allStreets) { + String key = m.getPartialName(); + Integer count = countMap1.get(key); + if (count == null) + countMap1.put(key, 1); + else + countMap1.put(key, 1 + count); + key = m.getInitialPart(); + if (!key.isEmpty()) { + count = countMap2.get(key); + if (count == null) + countMap2.put(key, 1); + else + countMap2.put(key, 1 + count); + } + } + + // remove keys which are rare + countMap1.entrySet().removeIf(e -> e.getValue() <= 3); + countMap2.entrySet().removeIf(e -> e.getValue() <= 3); + Map cache1 = new HashMap<>(); + Map cache2 = new HashMap<>(); + + for (Mdr7Record m : allStreets) { + String key = m.getPartialName(); + Integer count = countMap1.get(key); + SortKey k1 = sort.createSortKey(m, key,0, (count == null) ? null : cache1); + key = m.getInitialPart(); + count = countMap2.get(key); + SortKey k2 = sort.createSortKey(m, key, m.getMapIndex(), (count == null) ? null : cache2); + streetKeys.add(new MultiSortKey<>(k1, k2, null)); + } + return streetKeys; + } + + /** + * Create sort keys for each entry in allStreets. Some keys appear only once, others appear very often. + * We only use a cache for those which appear often enough. + * @param sort the Sort instance + * @return list of keys + */ + private List> createSimpleSortKeys(Sort sort) { + List> streetKeys = new ArrayList<>(allStreets.size()); + // step1 : find out how often each key appears + HashMap countMap = new HashMap<>(); + for (Mdr7Record m : allStreets) { + String key = m.getPartialName(); + Integer count = countMap.get(key); + if (count == null) + countMap.put(key, 1); + else + countMap.put(key, 1 + count); + } + + // remove keys which are rare + countMap.entrySet().removeIf(e -> e.getValue() <= 3); + Map cache = new HashMap<>(); + + for (Mdr7Record m : allStreets) { + String key = m.getPartialName(); + Integer count = countMap.get(key); + streetKeys.add(sort.createSortKey(m, key,m.getMapIndex(), (count == null) ? null : cache)); + } + return streetKeys; + } + public void writeSectData(ImgFileWriter writer) { String lastName = null; String lastPartial = null; Index: src/uk/me/parabola/imgfmt/app/srt/Sort.java =================================================================== --- src/uk/me/parabola/imgfmt/app/srt/Sort.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/srt/Sort.java (working copy) @@ -22,6 +22,7 @@ import java.text.CollationKey; import java.text.Collator; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -165,6 +166,9 @@ * @return A sort key. */ public SortKey createSortKey(T object, String s, int second, Map cache) { + if (s.length() == 0) + return new SrtSortKey<>(object, ZERO_KEY, second); + // If there is a cache then look up and return the key. // This is primarily for memory management, not for speed. byte[] key; @@ -194,14 +198,16 @@ // We need +1 for the null bytes, we also +2 for a couple of expanded characters. For a complete // german map this was always enough in tests. key = new byte[(chars.length + 1 + 2) * 4]; + int needed = 0; try { - fillCompleteKey(chars, key); + needed = fillCompleteKey(chars, key); } catch (ArrayIndexOutOfBoundsException e) { // Ok try again with the max possible key size allocated. key = new byte[(chars.length+1) * 4 * maxExpSize]; - fillCompleteKey(chars, key); + needed = fillCompleteKey(chars, key); } - + if ((key.length >> 3) > (needed >> 3)) + key = Arrays.copyOf(key, needed); if (cache != null) cache.put(s, key); @@ -240,14 +246,16 @@ // We need +1 for the null bytes, we also +2 for a couple of expanded characters. For a complete // german map this was always enough in tests. key = new byte[(encText.length + 1 + 2) * 4]; + int needed = 0; try { - fillCompleteKey(encText, key); + needed = fillCompleteKey(encText, key); } catch (ArrayIndexOutOfBoundsException e) { // Ok try again with the max possible key size allocated. key = new byte[encText.length * 4 * maxExpSize + 4]; - fillCompleteKey(encText, key); + needed = fillCompleteKey(encText, key); } - + if ((key.length >> 3) > (needed >> 3)) + key = Arrays.copyOf(key, needed); if (cache != null) cache.put(label, key); @@ -284,11 +292,12 @@ * * @param bVal The string for which we are creating the sort key. * @param key The sort key. This will be filled in. + * @return the needed number of bytes in case the buffer was large enough */ - private void fillCompleteKey(char[] bVal, byte[] key) { + private int fillCompleteKey(char[] bVal, byte[] key) { int start = fillKey(Collator.PRIMARY, bVal, key, 0); start = fillKey(Collator.SECONDARY, bVal, key, start); - fillKey(Collator.TERTIARY, bVal, key, start); + return fillKey(Collator.TERTIARY, bVal, key, start); } /**