Hi
In investigating another problem I discovered that an index created for
tiles in Greece (and it would also apply to any other place where
non-latin letters are used) does not work for finding streets.
You can select them but clicking on Find usually does not find them.
This is because the transliteration from Greek characters to Roman
characters occurs after the labels are sorted for the NET section.
This results in them being completely mis-sorted in NET.
The attached patch does the transliteration first.
It doesn't much affect maps of western European countries, but it would
be good to get wider testing before committing it.
Maps with ascii labels are sometimes a little smaller, because name that
are different but transliterate to the same string used to be duplicated
but with this patch they are not.
..Steve
Index: src/uk/me/parabola/imgfmt/app/labelenc/LatinEncoder.java
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/LatinEncoder.java (revision 1650)
+++ src/uk/me/parabola/imgfmt/app/labelenc/LatinEncoder.java (revision )
@@ -20,7 +20,6 @@
* @author Steve Ratcliffe
*/
public class LatinEncoder extends BaseEncoder implements CharacterEncoder {
- private final Transliterator trans = new TableTransliterator("latin1");
private final Charset latinCharset = Charset.forName("latin1");
public EncodedText encodeText(String t) {
@@ -32,8 +31,7 @@
// Need to add a null character at the end of the string for this format.
String zText = text + "\000";
- String s = trans.transliterate(zText);
- byte[] chars = s.getBytes(latinCharset);
+ byte[] chars = zText.getBytes(latinCharset);
return new EncodedText(chars, chars.length);
}
}
Index: src/uk/me/parabola/imgfmt/app/labelenc/CodeFunctions.java
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/CodeFunctions.java (revision 1650)
+++ src/uk/me/parabola/imgfmt/app/labelenc/CodeFunctions.java (revision )
@@ -29,6 +29,7 @@
private int encodingType;
private CharacterEncoder encoder;
private CharacterDecoder decoder;
+ private Transliterator transliterator = new TableTransliterator("ascii");
protected void setEncoder(CharacterEncoder encoder) {
this.encoder = encoder;
@@ -62,6 +63,14 @@
this.codepage = codepage;
}
+ public Transliterator getTransliterator() {
+ return transliterator;
+ }
+
+ public void setTransliterator(Transliterator transliterator) {
+ this.transliterator = transliterator;
+ }
+
/**
* Create a CharacterEncoder for the given charset option. Note that this
* routine also writes to the lblHeader parameter to set the encoding type.
@@ -74,10 +83,12 @@
if ("ascii".equals(charset)) {
funcs.setEncodingType(ENCODING_FORMAT6);
funcs.setEncoder(new Format6Encoder());
+ funcs.setTransliterator(getDefaultTransliterator());
funcs.setDecoder(new Format6Decoder());
} else if ("latin1".equals(charset)) {
funcs.setEncodingType(ENCODING_FORMAT9);
funcs.setEncoder(new LatinEncoder());
+ funcs.setTransliterator(new TableTransliterator("latin1"));
funcs.setDecoder(new AnyCharsetDecoder("cp1252"));
funcs.setCodepage(1252);
} else if ("unicode".equals(charset)) {
@@ -104,6 +115,7 @@
*/
private static void guessCodepage(CodeFunctions funcs, String charset) {
String cs = charset.toLowerCase();
+ Transliterator transliterator = getDefaultTransliterator();
if (cs.startsWith("cp")) {
try {
funcs.setCodepage(Integer.parseInt(charset.substring(2)));
@@ -118,7 +130,9 @@
}
} else if (cs.equals("latin1")) {
funcs.setCodepage(1252);
+ transliterator = new TableTransliterator("latin1");
}
+ funcs.setTransliterator(transliterator);
}
/**
@@ -152,4 +166,10 @@
public static CharacterDecoder getDefaultDecoder() {
return new Format6Decoder();
}
+
+ public static Transliterator getDefaultTransliterator() {
+ TableTransliterator ascii = new TableTransliterator("ascii");
+ ascii.forceUppercase(true);
+ return ascii;
-}
+ }
+}
Index: test/uk/me/parabola/imgfmt/app/labelenc/CodeFunctionsTest.java
===================================================================
--- test/uk/me/parabola/imgfmt/app/labelenc/CodeFunctionsTest.java (revision 1519)
+++ test/uk/me/parabola/imgfmt/app/labelenc/CodeFunctionsTest.java (revision )
@@ -58,7 +58,8 @@
CodeFunctions functions = CodeFunctions.createEncoderForLBL(6);
CharacterEncoder encoder = functions.getEncoder();
- EncodedText text = encoder.encodeText("KörnerstraÃe, VelkomezeÅÃÄská, Skólavörðustigur");
+ Transliterator transliterator = functions.getTransliterator();
+ EncodedText text = encoder.encodeText(transliterator.transliterate("KörnerstraÃe, VelkomezeÅÃÄská, Skólavörðustigur"));
CharacterDecoder decoder = functions.getDecoder();
byte[] ctext = text.getCtext();
@@ -79,7 +80,8 @@
CodeFunctions functions = CodeFunctions.createEncoderForLBL("latin1");
CharacterEncoder encoder = functions.getEncoder();
- EncodedText text = encoder.encodeText("KörnerstraÃe, VelkomezeÅÃÄská, Skólavörðustigur");
+ Transliterator transliterator = functions.getTransliterator();
+ EncodedText text = encoder.encodeText(transliterator.transliterate("KörnerstraÃe, VelkomezeÅÃÄská, Skólavörðustigur"));
CharacterDecoder decoder = functions.getDecoder();
byte[] ctext = text.getCtext();
Index: src/uk/me/parabola/imgfmt/app/labelenc/Format6Encoder.java
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/Format6Encoder.java (revision 1652)
+++ src/uk/me/parabola/imgfmt/app/labelenc/Format6Encoder.java (revision )
@@ -48,8 +48,6 @@
"xxxxxxxxxx:;<=>?" + // 0x10-0x1F
"xxxxxxxxxxx[\\]^_"; // 0x20-0x2F
- private final Transliterator transliterator = new TableTransliterator("ascii");
-
/**
* Encode the text into the 6 bit format. See the class level notes.
*
@@ -69,7 +67,7 @@
byte[] buf = new byte[2 * s.length() + 4];
int off = 0;
- for (char c : transliterator.transliterate(s).toCharArray()) {
+ for (char c : s.toCharArray()) {
if (c == ' ') {
put6(buf, off++, 0);
Index: src/uk/me/parabola/imgfmt/app/labelenc/TableTransliterator.java
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/TableTransliterator.java (revision 1751)
+++ src/uk/me/parabola/imgfmt/app/labelenc/TableTransliterator.java (revision )
@@ -18,6 +18,7 @@
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Arrays;
+import java.util.Locale;
import uk.me.parabola.log.Logger;
@@ -35,6 +36,7 @@
private final String[][] rows = new String[256][];
private final boolean useLatin;
+ private boolean forceUppercase;
public TableTransliterator(String targetCharset) {
if (targetCharset.equals("latin1"))
@@ -46,11 +48,15 @@
/**
* Convert a string into a string that uses only ascii characters.
*
- * @param s The original string. It can use any unicode character.
+ * @param s The original string. It can use any unicode character. Can be null in which case null will
+ * be returned.
* @return A string that uses only ascii characters that is a transcription or
* transliteration of the original string.
*/
public String transliterate(String s) {
+ if (s == null)
+ return null;
+
StringBuilder sb = new StringBuilder(s.length() + 5);
for (char c : s.toCharArray()) {
if (c <= (useLatin? 0xff: 0x7f)) {
@@ -65,9 +71,16 @@
}
}
- return sb.toString();
+ String text = sb.toString();
+ if (forceUppercase)
+ text = text.toUpperCase(Locale.ENGLISH);
+ return text;
}
+ public void forceUppercase(boolean uc) {
+ forceUppercase = uc;
+ }
+
/**
* Load one row of characters. This means unicode characters that are of the
* form U+RRXX where RR is the row.
@@ -127,9 +140,13 @@
// The first field must look like 'U+RRXX', we extract the XX part
int index = Integer.parseInt(upoint.substring(4), 16);
- if (newRow[index].equals("?"))
+ if (newRow[index].equals("?")) {
+ if (forceUppercase)
+ newRow[index] = translation.toUpperCase(Locale.ENGLISH);
+ else
- newRow[index] = translation;
- }
+ newRow[index] = translation;
+ }
+ }
} catch (IOException e) {
log.error("Could not read character translation table");
}
Index: src/uk/me/parabola/imgfmt/app/labelenc/Transliterator.java
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/Transliterator.java (revision 1519)
+++ src/uk/me/parabola/imgfmt/app/labelenc/Transliterator.java (revision )
@@ -19,11 +19,18 @@
*/
public interface Transliterator {
/**
- * Convert a string into a string that uses only ascii characters.
+ * Convert a string into a string that uses only ascii or latin1 characters.
*
- * @param s The original string. It can use any unicode character.
+ * @param s The original string. It can use any unicode character. Can be null in which
+ * case null will be returned.
* @return A string that uses a restricted subset of characters (ascii or
* latin) that is a transliterated form of the input string.
*/
public String transliterate(String s);
+
+ /**
+ * Force the use of uppercase in this transliterator.
+ * Note that it is normal to set this.
+ */
+ public void forceUppercase(boolean uc);
}
Index: src/uk/me/parabola/imgfmt/app/lbl/LBLFile.java
===================================================================
--- src/uk/me/parabola/imgfmt/app/lbl/LBLFile.java (revision 1870)
+++ src/uk/me/parabola/imgfmt/app/lbl/LBLFile.java (revision )
@@ -17,7 +17,6 @@
package uk.me.parabola.imgfmt.app.lbl;
import java.util.HashMap;
-import java.util.Locale;
import java.util.Map;
import uk.me.parabola.imgfmt.Utils;
@@ -29,7 +28,7 @@
import uk.me.parabola.imgfmt.app.labelenc.BaseEncoder;
import uk.me.parabola.imgfmt.app.labelenc.CharacterEncoder;
import uk.me.parabola.imgfmt.app.labelenc.CodeFunctions;
-import uk.me.parabola.imgfmt.app.labelenc.Format6Encoder;
+import uk.me.parabola.imgfmt.app.labelenc.Transliterator;
import uk.me.parabola.imgfmt.app.srt.Sort;
import uk.me.parabola.imgfmt.app.trergn.Subdivision;
import uk.me.parabola.imgfmt.fs.ImgChannel;
@@ -49,6 +48,7 @@
private static final Logger log = Logger.getLogger(LBLFile.class);
private CharacterEncoder textEncoder = CodeFunctions.getDefaultEncoder();
+ private Transliterator transliterator = CodeFunctions.getDefaultTransliterator();
private final Map<String, Label> labelCache = new HashMap<String, Label>();
@@ -101,27 +101,24 @@
lblHeader.setEncodingType(cfuncs.getEncodingType());
textEncoder = cfuncs.getEncoder();
+ transliterator = cfuncs.getTransliterator();
if (forceUpper && textEncoder instanceof BaseEncoder) {
BaseEncoder baseEncoder = (BaseEncoder) textEncoder;
baseEncoder.setUpperCase(true);
}
+ if (forceUpper)
+ transliterator.forceUppercase(true);
}
/**
* Add a new label with the given text. Labels are shared, so that identical
* text is always represented by the same label.
*
- * @param text The text of the label, it will be in uppercase.
+ * @param inText The text of the label, it will be in uppercase.
* @return A reference to the created label.
*/
- public Label newLabel(String text) {
- // if required, fold case now so that labelCache doesn't
- // contain multiple labels that only differ in letter case
- if(text != null &&
- (textEncoder instanceof Format6Encoder ||
- textEncoder instanceof BaseEncoder &&
- ((BaseEncoder)textEncoder).isUpperCase()))
- text = text.toUpperCase(Locale.ENGLISH);
+ public Label newLabel(String inText) {
+ String text = transliterator.transliterate(inText);
Label l = labelCache.get(text);
if (l == null) {
l = new Label(text);
_______________________________________________
mkgmap-dev mailing list
[email protected]
http://www.mkgmap.org.uk/mailman/listinfo/mkgmap-dev
