Index: src/uk/me/parabola/imgfmt/app/mdr/MDRFile.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/MDRFile.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/MDRFile.java (working copy) @@ -296,6 +296,8 @@ mdr18.setPoiTypes(mdr19.getPoiTypes()); mdr19.release(); writeSection(writer, 18, mdr18); + } else { + mdr19.release(); } writeSection(writer, 10, mdr10); Index: src/uk/me/parabola/imgfmt/app/mdr/Mdr20.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/Mdr20.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/Mdr20.java (working copy) @@ -15,6 +15,7 @@ import java.text.Collator; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -50,9 +51,9 @@ * Also have to set the record number of the first record in this section * on the city. * - * @param inStreets The list of streets from mdr7, must have Mdr7.index set. + * @param inStreets The collection of streets from mdr7, must have Mdr7.index set. */ - public void buildFromStreets(List inStreets) { + public void buildFromStreets(Collection inStreets) { Sort sort = getConfig().getSort(); // Use a key cache because there are a large number of street names but a much smaller number Index: src/uk/me/parabola/imgfmt/app/mdr/Mdr21.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/Mdr21.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/Mdr21.java (working copy) @@ -13,6 +13,7 @@ package uk.me.parabola.imgfmt.app.mdr; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -37,9 +38,9 @@ * We need to sort the streets by the name of their region. Within a region * group the streets are ordered by their own index. * - * @param inStreets The list of streets from mdr7. + * @param inStreets The collection of streets from mdr7. */ - public void buildFromStreets(List inStreets) { + public void buildFromStreets(Collection inStreets) { Sort sort = getConfig().getSort(); List> keys = new ArrayList<>(); Index: src/uk/me/parabola/imgfmt/app/mdr/Mdr22.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/Mdr22.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/Mdr22.java (working copy) @@ -14,6 +14,7 @@ package uk.me.parabola.imgfmt.app.mdr; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -43,9 +44,9 @@ * Also have to set the record number of the first record in this section * on the city. * - * @param inStreets The list of streets from mdr7. + * @param inStreets The collection of streets from mdr7. */ - public void buildFromStreets(List inStreets) { + public void buildFromStreets(Collection inStreets) { Sort sort = getConfig().getSort(); List> keys = new ArrayList<>(); Index: src/uk/me/parabola/imgfmt/app/mdr/Mdr7.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/Mdr7.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/Mdr7.java (working copy) @@ -13,12 +13,18 @@ package uk.me.parabola.imgfmt.app.mdr; import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import uk.me.parabola.imgfmt.MapFailedException; import uk.me.parabola.imgfmt.app.ImgFileWriter; -import uk.me.parabola.imgfmt.app.srt.MultiSortKey; +import uk.me.parabola.imgfmt.app.srt.DoubleSortKey; import uk.me.parabola.imgfmt.app.srt.Sort; import uk.me.parabola.imgfmt.app.srt.SortKey; @@ -41,23 +47,28 @@ private final boolean isMulti; private final boolean splitName; - private List allStreets = new ArrayList<>(); + private Collection allStreets = new LinkedHashSet<>(); private List streets = new ArrayList<>(); private final int u2size = 1; + private Set exclNames; public Mdr7(MdrConfig config) { setConfig(config); Sort sort = config.getSort(); splitName = config.isSplitName(); + exclNames = config.getMdr7Excl(); codepage = sort.getCodepage(); isMulti = sort.isMulti(); + } public void addStreet(int mapId, String name, int lblOffset, int strOff, Mdr5Record mdrCity) { if (name.isEmpty()) return; - + if (exclNames.contains(name)) + return; + // Find a name prefix, which is either a shield or a word ending 0x1e. We are treating // a shield as a prefix of length one. int prefix = 0; @@ -85,6 +96,9 @@ st.setCity(mdrCity); st.setPrefixOffset((byte) prefix); st.setSuffixOffset((byte) suffix); + if (exclNames.contains(st.getInitialPart())) + return; + allStreets.add(st); if (!splitName) @@ -123,7 +137,8 @@ st.setPrefixOffset((byte) prefix); st.setSuffixOffset((byte) suffix); //System.out.println(st.getName() + ": add partial " + st.getPartialName()); - allStreets.add(st); + if (!exclNames.contains(st.getPartialName())) + allStreets.add(st); start = false; } @@ -167,14 +182,13 @@ * as it requires a lot of heap to store the sort keys. */ protected void preWriteImpl() { + allStreets = new ArrayList<>(allStreets); Sort sort = getConfig().getSort(); - List> sortedStreets = new ArrayList<>(allStreets.size()); - for (Mdr7Record m : allStreets) { - sortedStreets.add(new MultiSortKey<>( - sort.createSortKey(m, m.getPartialName()), - sort.createSortKey(m, m.getInitialPart(), m.getMapIndex()), - null)); - } + List> sortedStreets; + if (splitName) + sortedStreets = createDoubleSortKeys (sort); + else + sortedStreets = createSimpleSortKeys (sort); Collections.sort(sortedStreets); // De-duplicate the street names so that there is only one entry @@ -200,8 +214,103 @@ // release memory sortedStreets.set(i, null); } + return; } + /** + * Create sort keys for each entry in allStreets. Some keys appear only once, others appear very often. + * We only use a cache for those which appear often enough. + * @param sort the Sort instance + * @return list of keys + */ + private List> createDoubleSortKeys(Sort sort) { + // step1 : find out how often each key appears + HashMap countMap1 = new HashMap<>(); + HashMap countMap2 = new HashMap<>(); + for (Mdr7Record m : allStreets) { + String key = m.getPartialName(); + Integer count = countMap1.get(key); + if (count == null) + countMap1.put(key, 1); + else + countMap1.put(key, 1 + count); + key = m.getInitialPart(); + if (!key.isEmpty()) { + count = countMap2.get(key); + if (count == null) + countMap2.put(key, 1); + else + countMap2.put(key, 1 + count); + } + } + // convert maps to BitSets to reduce memory + BitSet doCache1 = new BitSet(allStreets.size()); + BitSet doCache2 = new BitSet(allStreets.size()); + int pos = 0; + for (Mdr7Record m : allStreets) { + if (countMap1.get(m.getPartialName()) > 3) + doCache1.set(pos); + String key = m.getInitialPart(); + if (!key.isEmpty()) { + if (countMap2.get(key) > 3) + doCache2.set(pos); + } + pos++; + } + countMap1 = null; + countMap2 = null; + + // we know now where to use the caches, let's create the sort keys + Map cache1 = new HashMap<>(); + Map cache2 = new HashMap<>(); + List> streetKeys = new ArrayList<>(allStreets.size()); + pos = 0; + for (Mdr7Record m : allStreets) { + String key = m.getPartialName(); + SortKey k1 = sort.createSortKey(m, key,0, doCache1.get(pos) ? cache1 : null); + key = m.getInitialPart(); + SortKey k2 = sort.createSortKey(m, key, m.getMapIndex(), doCache2.get(pos) ? cache2 : null); + streetKeys.add(new DoubleSortKey<>(k1, k2)); + pos++; + } + return streetKeys; + } + + /** + * Create sort keys for each entry in allStreets. Some keys appear only once, others appear very often. + * We only use a cache for those which appear often enough. + * @param sort the Sort instance + * @return list of keys + */ + private List> createSimpleSortKeys(Sort sort) { + // step1 : find out how often each key appears + HashMap countMap = new HashMap<>(); + for (Mdr7Record m : allStreets) { + String key = m.getName(); + Integer count = countMap.get(key); + if (count == null) + countMap.put(key, 1); + else + countMap.put(key, 1 + count); + } + BitSet doCache = new BitSet(allStreets.size()); + int pos = 0; + for (Mdr7Record m : allStreets) { + if (countMap.get(m.getName()) > 3) + doCache.set(pos); + pos++; + } + countMap = null; + + Map cache = new HashMap<>(); + List> streetKeys = new ArrayList<>(allStreets.size()); + pos = 0; + for (Mdr7Record m : allStreets) { + streetKeys.add(sort.createSortKey(m, m.getName(), m.getMapIndex(), doCache.get(pos++) ? cache : null)); + } + return streetKeys; + } + public void writeSectData(ImgFileWriter writer) { String lastName = null; String lastPartial = null; @@ -324,8 +433,8 @@ return name.substring(0, endIndex); } - public List getStreets() { - return Collections.unmodifiableList(allStreets); + public Collection getStreets() { + return Collections.unmodifiableCollection(allStreets); } public List getSortedStreets() { Index: src/uk/me/parabola/imgfmt/app/mdr/Mdr7Record.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/Mdr7Record.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/Mdr7Record.java (working copy) @@ -117,4 +117,56 @@ public String getInitialPart() { return name.substring(0, (nameOffset & 0xff)); } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((city == null) ? 0 : city.hashCode()); + result = prime * result + index; + result = prime * result + labelOffset; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + nameOffset; + result = prime * result + outNameOffset; + result = prime * result + prefixOffset; + result = prime * result + stringOffset; + result = prime * result + suffixOffset; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Mdr7Record other = (Mdr7Record) obj; + if (city == null) { + if (other.city != null) + return false; + } else if (!city.equals(other.city)) + return false; + if (index != other.index) + return false; + if (labelOffset != other.labelOffset) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + if (nameOffset != other.nameOffset) + return false; + if (outNameOffset != other.outNameOffset) + return false; + if (prefixOffset != other.prefixOffset) + return false; + if (stringOffset != other.stringOffset) + return false; + if (suffixOffset != other.suffixOffset) + return false; + return true; + } } Index: src/uk/me/parabola/imgfmt/app/mdr/MdrConfig.java =================================================================== --- src/uk/me/parabola/imgfmt/app/mdr/MdrConfig.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/mdr/MdrConfig.java (working copy) @@ -13,6 +13,10 @@ package uk.me.parabola.imgfmt.app.mdr; import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import uk.me.parabola.imgfmt.app.srt.Sort; @@ -33,7 +37,8 @@ private Sort sort; private File outputDir; private boolean splitName; - + private Set mdr7Excl = Collections.emptySet(); + /** * True if we are creating the file, rather than reading it. */ @@ -96,4 +101,17 @@ public boolean isSplitName() { return splitName; } + + public Set getMdr7Excl() { + return Collections.unmodifiableSet(mdr7Excl); + } + + public void setMdr7Excl(String s) { + if (s == null) + this.mdr7Excl = Collections.emptySet(); + else { + mdr7Excl = new HashSet<>(Arrays.asList(s.split(","))); + } + + } } Index: src/uk/me/parabola/imgfmt/app/srt/DoubleSortKey.java =================================================================== --- src/uk/me/parabola/imgfmt/app/srt/DoubleSortKey.java (nonexistent) +++ src/uk/me/parabola/imgfmt/app/srt/DoubleSortKey.java (working copy) @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2017. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 3 or + * version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +package uk.me.parabola.imgfmt.app.srt; + +/** + * Combines a pair of sort keys into one. The first is the primary sort and contains the + * actual object being sorted. + * + * @author Steve Ratcliffe + * @author Gerd Petermann + */ +public class DoubleSortKey implements SortKey { + private final SortKey key1; + private final SortKey key2; + + public DoubleSortKey(SortKey key1, SortKey key2) { + this.key1 = key1; + this.key2 = key2; + } + + public T getObject() { + return key1.getObject(); + } + + public int compareTo(SortKey o) { + DoubleSortKey other = (DoubleSortKey) o; + int res = key1.compareTo(other.key1); + if (res == 0) { + res = key2.compareTo(other.key2); + } + return res; + } +} Index: src/uk/me/parabola/imgfmt/app/srt/Sort.java =================================================================== --- src/uk/me/parabola/imgfmt/app/srt/Sort.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/srt/Sort.java (working copy) @@ -22,6 +22,7 @@ import java.text.CollationKey; import java.text.Collator; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -165,6 +166,9 @@ * @return A sort key. */ public SortKey createSortKey(T object, String s, int second, Map cache) { + if (s.length() == 0) + return new SrtSortKey<>(object, ZERO_KEY, second); + // If there is a cache then look up and return the key. // This is primarily for memory management, not for speed. byte[] key; @@ -194,14 +198,16 @@ // We need +1 for the null bytes, we also +2 for a couple of expanded characters. For a complete // german map this was always enough in tests. key = new byte[(chars.length + 1 + 2) * 4]; + int needed = 0; try { - fillCompleteKey(chars, key); + needed = fillCompleteKey(chars, key); } catch (ArrayIndexOutOfBoundsException e) { // Ok try again with the max possible key size allocated. key = new byte[(chars.length+1) * 4 * maxExpSize]; - fillCompleteKey(chars, key); + needed = fillCompleteKey(chars, key); } - + if ((key.length >> 3) > (needed >> 3)) + key = Arrays.copyOf(key, needed); if (cache != null) cache.put(s, key); @@ -240,14 +246,16 @@ // We need +1 for the null bytes, we also +2 for a couple of expanded characters. For a complete // german map this was always enough in tests. key = new byte[(encText.length + 1 + 2) * 4]; + int needed = 0; try { - fillCompleteKey(encText, key); + needed = fillCompleteKey(encText, key); } catch (ArrayIndexOutOfBoundsException e) { // Ok try again with the max possible key size allocated. key = new byte[encText.length * 4 * maxExpSize + 4]; - fillCompleteKey(encText, key); + needed = fillCompleteKey(encText, key); } - + if ((key.length >> 3) > (needed >> 3)) + key = Arrays.copyOf(key, needed); if (cache != null) cache.put(label, key); @@ -284,11 +292,12 @@ * * @param bVal The string for which we are creating the sort key. * @param key The sort key. This will be filled in. + * @return the needed number of bytes in case the buffer was large enough */ - private void fillCompleteKey(char[] bVal, byte[] key) { + private int fillCompleteKey(char[] bVal, byte[] key) { int start = fillKey(Collator.PRIMARY, bVal, key, 0); start = fillKey(Collator.SECONDARY, bVal, key, start); - fillKey(Collator.TERTIARY, bVal, key, start); + return fillKey(Collator.TERTIARY, bVal, key, start); } /** Index: src/uk/me/parabola/imgfmt/app/srt/SrtSortKey.java =================================================================== --- src/uk/me/parabola/imgfmt/app/srt/SrtSortKey.java (revision 3873) +++ src/uk/me/parabola/imgfmt/app/srt/SrtSortKey.java (working copy) @@ -39,17 +39,18 @@ public int compareTo(SortKey o) { SrtSortKey other = (SrtSortKey) o; - int length = Math.min(this.key.length, other.key.length); - for (int i = 0; i < length; i++) { - int k1 = this.key[i] & 0xff; - int k2 = other.key[i] & 0xff; - if (k1 < k2) { - return -1; - } else if (k1 > k2) { - return 1; + if (key != other.key) { + int length = Math.min(this.key.length, other.key.length); + for (int i = 0; i < length; i++) { + int k1 = this.key[i] & 0xff; + int k2 = other.key[i] & 0xff; + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } } } - //if (this.key.length < other.key.length) // return -1; //else if (this.key.length > other.key.length) Index: src/uk/me/parabola/mkgmap/combiners/MdrBuilder.java =================================================================== --- src/uk/me/parabola/mkgmap/combiners/MdrBuilder.java (revision 3873) +++ src/uk/me/parabola/mkgmap/combiners/MdrBuilder.java (working copy) @@ -112,6 +112,7 @@ config.setOutputDir(outputDir); config.setSort(sort); config.setSplitName(args.get("split-name-index", false)); + config.setMdr7Excl(args.get("mdr7-excl", null)); // Wrap the MDR channel with the MDRFile object mdrFile = new MDRFile(mdrChan, config);