Index: src/uk/me/parabola/imgfmt/app/mdr/MDRFile.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/MDRFile.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/MDRFile.java (working copy) @@ -296,6 +296,8 @@ mdr18.setPoiTypes(mdr19.getPoiTypes()); mdr19.release(); writeSection(writer, 18, mdr18); + } else { + mdr19.release(); } writeSection(writer, 10, mdr10); Index: src/uk/me/parabola/imgfmt/app/mdr/Mdr7.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/Mdr7.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/Mdr7.java (working copy) @@ -14,7 +14,10 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; import uk.me.parabola.imgfmt.MapFailedException; import uk.me.parabola.imgfmt.app.ImgFileWriter; @@ -45,19 +48,24 @@ private List streets = new ArrayList<>(); private final int u2size = 1; + private Set exclNames; public Mdr7(MdrConfig config) { setConfig(config); Sort sort = config.getSort(); splitName = config.isSplitName(); + exclNames = config.getMdr7Excl(); codepage = sort.getCodepage(); isMulti = sort.isMulti(); + } public void addStreet(int mapId, String name, int lblOffset, int strOff, Mdr5Record mdrCity) { if (name.isEmpty()) return; - + if (exclNames.contains(name)) + return; + // Find a name prefix, which is either a shield or a word ending 0x1e. We are treating // a shield as a prefix of length one. int prefix = 0; @@ -85,6 +93,9 @@ st.setCity(mdrCity); st.setPrefixOffset((byte) prefix); st.setSuffixOffset((byte) suffix); + if (exclNames.contains(st.getInitialPart())) + return; + allStreets.add(st); if (!splitName) @@ -123,7 +134,8 @@ st.setPrefixOffset((byte) prefix); st.setSuffixOffset((byte) suffix); //System.out.println(st.getName() + ": add partial " + st.getPartialName()); - allStreets.add(st); + if (!exclNames.contains(st.getPartialName())) + allStreets.add(st); start = false; } @@ -168,13 +180,11 @@ */ protected void preWriteImpl() { Sort sort = getConfig().getSort(); - List> sortedStreets = new ArrayList<>(allStreets.size()); - for (Mdr7Record m : allStreets) { - sortedStreets.add(new MultiSortKey<>( - sort.createSortKey(m, m.getPartialName()), - sort.createSortKey(m, m.getInitialPart(), m.getMapIndex()), - null)); - } + List> sortedStreets; + if (splitName) + sortedStreets = createMultiSortKeys (sort); + else + sortedStreets = createSimpleSortKeys (sort); Collections.sort(sortedStreets); // De-duplicate the street names so that there is only one entry @@ -200,8 +210,91 @@ // release memory sortedStreets.set(i, null); } + return; } + /** + * Create sort keys for each entry in allStreets. Some keys appear only once, others appear very often. + * We only use a cache for those which appear often enough. + * @param sort the Sort instance + * @return list of keys + */ + private List> createMultiSortKeys(Sort sort) { + List> streetKeys = new ArrayList<>(allStreets.size()); + // step1 : find out how often each key appears + HashMap countMap1 = new HashMap<>(); + HashMap countMap2 = new HashMap<>(); + for (Mdr7Record m : allStreets) { + String key = m.getPartialName(); + Integer count = countMap1.get(key); + if (count == null) + countMap1.put(key, 1); + else + countMap1.put(key, 1 + count); + key = m.getInitialPart(); + if (!key.isEmpty()) { + count = countMap2.get(key); + if (count == null) + countMap2.put(key, 1); + else + countMap2.put(key, 1 + count); + } + } + + // remove keys which are rare + countMap1.entrySet().removeIf(e -> e.getValue() <= 3); + countMap2.entrySet().removeIf(e -> e.getValue() <= 3); + // report values which make > 1 percent + final int threshold = allStreets.size() / 100; + countMap1.entrySet().stream().filter(e -> e.getValue() > threshold) + .forEach(e -> System.out.println("Street index (Mdr7) '" + e.getKey() + "' : " + + 10000 * e.getValue() / allStreets.size() / 100.0 + "%")); + Map cache1 = new HashMap<>(); + Map cache2 = new HashMap<>(); + + for (Mdr7Record m : allStreets) { + String key = m.getPartialName(); + Integer count = countMap1.get(key); + SortKey k1 = sort.createSortKey(m, key,0, (count == null) ? null : cache1); + key = m.getInitialPart(); + count = countMap2.get(key); + SortKey k2 = sort.createSortKey(m, key, m.getMapIndex(), (count == null) ? null : cache2); + streetKeys.add(new MultiSortKey<>(k1, k2, null)); + } + return streetKeys; + } + + /** + * Create sort keys for each entry in allStreets. Some keys appear only once, others appear very often. + * We only use a cache for those which appear often enough. + * @param sort the Sort instance + * @return list of keys + */ + private List> createSimpleSortKeys(Sort sort) { + List> streetKeys = new ArrayList<>(allStreets.size()); + // step1 : find out how often each key appears + HashMap countMap = new HashMap<>(); + for (Mdr7Record m : allStreets) { + String key = m.getPartialName(); + Integer count = countMap.get(key); + if (count == null) + countMap.put(key, 1); + else + countMap.put(key, 1 + count); + } + + // remove keys which are rare + countMap.entrySet().removeIf(e -> e.getValue() <= 3); + Map cache = new HashMap<>(); + + for (Mdr7Record m : allStreets) { + String key = m.getPartialName(); + Integer count = countMap.get(key); + streetKeys.add(sort.createSortKey(m, key,m.getMapIndex(), (count == null) ? null : cache)); + } + return streetKeys; + } + public void writeSectData(ImgFileWriter writer) { String lastName = null; String lastPartial = null; Index: src/uk/me/parabola/imgfmt/app/mdr/MdrConfig.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/MdrConfig.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/MdrConfig.java (working copy) @@ -13,6 +13,10 @@ package uk.me.parabola.imgfmt.app.mdr; import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import uk.me.parabola.imgfmt.app.srt.Sort; @@ -33,7 +37,8 @@ private Sort sort; private File outputDir; private boolean splitName; - + private Set mdr7Excl; + /** * True if we are creating the file, rather than reading it. */ @@ -96,4 +101,17 @@ public boolean isSplitName() { return splitName; } + + public Set getMdr7Excl() { + return Collections.unmodifiableSet(mdr7Excl); + } + + public void setMdr7Excl(String s) { + if (s == null) + this.mdr7Excl = Collections.emptySet(); + else { + mdr7Excl = new HashSet<>(Arrays.asList(s.split(","))); + } + + } } Index: src/uk/me/parabola/imgfmt/app/srt/Sort.java =================================================================== --- src/uk/me/parabola/imgfmt/app/srt/Sort.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/srt/Sort.java (working copy) @@ -22,6 +22,7 @@ import java.text.CollationKey; import java.text.Collator; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -165,6 +166,9 @@ * @return A sort key. */ public SortKey createSortKey(T object, String s, int second, Map cache) { + if (s.length() == 0) + return new SrtSortKey<>(object, ZERO_KEY, second); + // If there is a cache then look up and return the key. // This is primarily for memory management, not for speed. byte[] key; @@ -194,14 +198,16 @@ // We need +1 for the null bytes, we also +2 for a couple of expanded characters. For a complete // german map this was always enough in tests. key = new byte[(chars.length + 1 + 2) * 4]; + int needed = 0; try { - fillCompleteKey(chars, key); + needed = fillCompleteKey(chars, key); } catch (ArrayIndexOutOfBoundsException e) { // Ok try again with the max possible key size allocated. key = new byte[(chars.length+1) * 4 * maxExpSize]; - fillCompleteKey(chars, key); + needed = fillCompleteKey(chars, key); } - + if ((key.length >> 3) > (needed >> 3)) + key = Arrays.copyOf(key, needed); if (cache != null) cache.put(s, key); @@ -240,14 +246,16 @@ // We need +1 for the null bytes, we also +2 for a couple of expanded characters. For a complete // german map this was always enough in tests. key = new byte[(encText.length + 1 + 2) * 4]; + int needed = 0; try { - fillCompleteKey(encText, key); + needed = fillCompleteKey(encText, key); } catch (ArrayIndexOutOfBoundsException e) { // Ok try again with the max possible key size allocated. key = new byte[encText.length * 4 * maxExpSize + 4]; - fillCompleteKey(encText, key); + needed = fillCompleteKey(encText, key); } - + if ((key.length >> 3) > (needed >> 3)) + key = Arrays.copyOf(key, needed); if (cache != null) cache.put(label, key); @@ -284,11 +292,12 @@ * * @param bVal The string for which we are creating the sort key. * @param key The sort key. This will be filled in. + * @return the needed number of bytes in case the buffer was large enough */ - private void fillCompleteKey(char[] bVal, byte[] key) { + private int fillCompleteKey(char[] bVal, byte[] key) { int start = fillKey(Collator.PRIMARY, bVal, key, 0); start = fillKey(Collator.SECONDARY, bVal, key, start); - fillKey(Collator.TERTIARY, bVal, key, start); + return fillKey(Collator.TERTIARY, bVal, key, start); } /** Index: src/uk/me/parabola/mkgmap/combiners/MdrBuilder.java =================================================================== --- src/uk/me/parabola/mkgmap/combiners/MdrBuilder.java (revision 3873) +++ src/uk/me/parabola/mkgmap/combiners/MdrBuilder.java (working copy) @@ -112,6 +112,7 @@ config.setOutputDir(outputDir); config.setSort(sort); config.setSplitName(args.get("split-name-index", false)); + config.setMdr7Excl(args.get("mdr7-excl", null)); // Wrap the MDR channel with the MDRFile object mdrFile = new MDRFile(mdrChan, config);