On 16/09/13 00:19, Andrzej Popowski wrote:
Well, yes, we get a map with all names like this. You can imagine how
silly it looks like. I have asked once without success, maybe this is
better opportunity to request again.
Could we have a proper transliteration in CP1250?
Attached patch is first attempt at this. Any character that cannot
be represented in the target character set will be transliterated
to ascii.
..Steve
Index: .idea/runConfigurations/mkgmap.xml
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- .idea/runConfigurations/mkgmap.xml (revision 2687)
+++ .idea/runConfigurations/mkgmap.xml (revision )
@@ -3,7 +3,7 @@
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" value="uk.me.parabola.mkgmap.main.Main" />
<option name="VM_PARAMETERS" value="-ea -Xmx1000m" />
- <option name="PROGRAM_PARAMETERS" value="--tdbfile --index --code-page=65001 --route u.osm" />
+ <option name="PROGRAM_PARAMETERS" value="--tdbfile --index --route /opt/data/uk-test-1.osm.gz" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" value="" />
\ No newline at end of file
Index: src/uk/me/parabola/imgfmt/app/labelenc/AnyCharsetEncoder.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/AnyCharsetEncoder.java (revision 2687)
+++ src/uk/me/parabola/imgfmt/app/labelenc/AnyCharsetEncoder.java (revision )
@@ -16,7 +16,13 @@
*/
package uk.me.parabola.imgfmt.app.labelenc;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
+import java.util.Arrays;
import java.util.Locale;
/**
@@ -27,20 +33,24 @@
*/
public class AnyCharsetEncoder extends BaseEncoder implements CharacterEncoder {
- private final Charset charSet;
+ private final CharsetEncoder encoder;
+ private final Transliterator transliterator;
- public AnyCharsetEncoder(String cs) {
+ public AnyCharsetEncoder(String cs, Transliterator transliterator) {
+ this.transliterator = transliterator;
prepareForCharacterSet(cs);
- if (isCharsetSupported())
- charSet = Charset.forName(cs);
- else
- charSet = null;
+ if (isCharsetSupported()) {
+ encoder = Charset.forName(cs).newEncoder();
+ encoder.onUnmappableCharacter(CodingErrorAction.REPORT);
+ } else {
+ encoder = null;
- }
+ }
+ }
public EncodedText encodeText(String text) {
if (text == null)
return NO_TEXT;
-
+
if (!isCharsetSupported())
return simpleEncode(text);
@@ -50,10 +60,33 @@
else
ucText = text;
- byte[] bytes = ucText.getBytes(charSet);
- byte[] res = new byte[bytes.length + 1];
- System.arraycopy(bytes, 0, res, 0, bytes.length);
+ byte[] bytes = new byte[(int) (ucText.length() * encoder.maxBytesPerChar()) + 10];
+ ByteBuffer bb = ByteBuffer.wrap(bytes);
+ CharBuffer charBuffer = CharBuffer.wrap(ucText);
+ CoderResult result;
+
+ do {
+ result = encoder.encode(charBuffer, bb, true);
+ if (result.isUnmappable()) {
+ char c = charBuffer.get();
+ String s = String.valueOf(c);
+
+ s = transliterator.transliterate(s);
+
+ for (int i = 0; i < s.length(); i++)
+ bb.put((byte) s.charAt(i));
+ }
+ } while (result != CoderResult.UNDERFLOW);
+
+ // We need it to be null terminated but also to trim any extra memory from the allocated
+ // buffer.
+ byte[] res = Arrays.copyOf(bytes, bb.position() + 1);
return new EncodedText(res, res.length);
+ }
+
+ public void setUpperCase(boolean upperCase) {
+ super.setUpperCase(upperCase);
+ transliterator.forceUppercase(upperCase);
}
}
Index: src/uk/me/parabola/imgfmt/app/srt/Sort.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- src/uk/me/parabola/imgfmt/app/srt/Sort.java (revision 2687)
+++ src/uk/me/parabola/imgfmt/app/srt/Sort.java (revision )
@@ -337,7 +337,7 @@
sort.charset = Charset.forName("ascii");
sort.encoder = sort.charset.newEncoder();
sort.setDescription("Default sort");
- sort.setCodepage(codepage == 0? 1252: codepage);
+ sort.setCodepage(codepage);
return sort;
}
Index: src/uk/me/parabola/imgfmt/app/labelenc/CodeFunctions.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/CodeFunctions.java (revision 2687)
+++ src/uk/me/parabola/imgfmt/app/labelenc/CodeFunctions.java (revision )
@@ -31,7 +31,6 @@
private int encodingType;
private CharacterEncoder encoder;
private CharacterDecoder decoder;
- private Transliterator transliterator = new TableTransliterator("ascii");
protected void setEncoder(CharacterEncoder encoder) {
this.encoder = encoder;
@@ -65,14 +64,6 @@
this.codepage = codepage;
}
- public Transliterator getTransliterator() {
- return transliterator;
- }
-
- public void setTransliterator(Transliterator transliterator) {
- this.transliterator = transliterator;
- }
-
/**
* Create a CharacterEncoder for the given charset option. Note that this
* routine also writes to the lblHeader parameter to set the encoding type.
@@ -81,129 +72,90 @@
*/
public static CodeFunctions createEncoderForLBL(String charset) {
CodeFunctions funcs = new CodeFunctions();
-
if ("ascii".equals(charset)) {
funcs.setEncodingType(ENCODING_FORMAT6);
funcs.setEncoder(new Format6Encoder());
- funcs.setTransliterator(getDefaultTransliterator());
funcs.setDecoder(new Format6Decoder());
- } else if ("latin1".equals(charset)) {
+ } else if ("cp1252".equals(charset) || "latin1".equals(charset)) {
funcs.setEncodingType(ENCODING_FORMAT9);
- funcs.setEncoder(new LatinEncoder());
- funcs.setTransliterator(new TableTransliterator("latin1"));
+ funcs.setEncoder(new AnyCharsetEncoder("cp1252", new TableTransliterator("latin1")));
funcs.setDecoder(new AnyCharsetDecoder("cp1252"));
funcs.setCodepage(1252);
- } else if ("unicode".equals(charset)) {
+ } else if ("cp65001".equals(charset) || "unicode".equals(charset)) {
funcs.setEncodingType(ENCODING_FORMAT10);
funcs.setEncoder(new Utf8Encoder());
funcs.setDecoder(new Utf8Decoder());
- } else if ("cp65001".equals(charset)) {
- funcs.setEncodingType(ENCODING_FORMAT10);
- funcs.setEncoder(new Utf8Encoder());
- funcs.setDecoder(new Utf8Decoder());
- funcs.setTransliterator(new NullTransliterator());
funcs.setCodepage(65001);
} else if ("simple8".equals(charset)) {
funcs.setEncodingType(ENCODING_FORMAT9);
funcs.setEncoder(new Simple8Encoder());
} else if ("cp932".equals(charset) || "ms932".equals(charset)) {
funcs.setEncodingType(ENCODING_FORMAT10);
- funcs.setEncoder(new AnyCharsetEncoder("ms932"));
+ funcs.setEncoder(new AnyCharsetEncoder("ms932", new SparseTransliterator("nomacron")));
funcs.setDecoder(new AnyCharsetDecoder("ms932"));
- funcs.setTransliterator(new SparseTransliterator("nomacron"));
funcs.setCodepage(932);
} else {
funcs.setEncodingType(ENCODING_FORMAT9);
funcs.setDecoder(new AnyCharsetDecoder(charset));
- funcs.setEncoder(new AnyCharsetEncoder(charset));
- guessCodepage(funcs, charset);
+ funcs.setEncoder(new AnyCharsetEncoder(charset, new TableTransliterator("ascii")));
+ funcs.setCodepage(guessCodepage(charset));
}
return funcs;
}
/**
+ * Sets encoding functions for a given format and code page. This is used
+ * when reading from an existing file.
+ *
+ * @param format The format from the lbl header.
+ * @param codePage The codepage found in the header.
+ * @return The various character set parameters that will be needed.
+ */
+ public static CodeFunctions createEncoderForLBL(int format, int codePage) {
+ CodeFunctions funcs;
+
+ if (format == ENCODING_FORMAT6) {
+ funcs = createEncoderForLBL("ascii");
+ } else {
+ funcs = createEncoderForLBL("cp" + codePage);
+ }
+
+ return funcs;
+ }
+
+ /**
* Guess the code page from the given charset. Only works with things
* like cp1252, windows-1252 and some well known ones.
- * @param funcs The code page functions.
* @param charset The charset that was given.
*/
- private static void guessCodepage(CodeFunctions funcs, String charset) {
+ private static int guessCodepage(String charset) {
String cs = charset.toLowerCase();
- Transliterator transliterator = new NullTransliterator();
if (cs.startsWith("cp")) {
try {
- funcs.setCodepage(Integer.parseInt(charset.substring(2)));
- if (cs.equals("cp1252"))
- transliterator = new TableTransliterator("latin1");
+ return Integer.parseInt(charset.substring(2));
} catch (NumberFormatException e) {
// wasn't in the right form
throw new ExitException("Invalid character set: " + cs);
}
} else if (cs.startsWith("windows-")) {
try {
- funcs.setCodepage(Integer.parseInt(charset.substring(8)));
+ return Integer.parseInt(charset.substring(8));
} catch (NumberFormatException e) {
// wasn't in the right form to guess
+ throw new ExitException("Invalid character set: " + cs);
}
} else if (cs.equals("latin1")) {
- funcs.setCodepage(1252);
- transliterator = new TableTransliterator("latin1");
+ return 1252;
}
- funcs.setTransliterator(transliterator);
+ return 0;
}
- /**
- * Sets encoding functions for a given format and code page. This is used
- * when reading from an existing file.
- *
- *
- * @param format The format from the lbl header.
- * @param codePage The codepage found in the header.
- * @return The various character set parameters that will be needed.
- */
- public static CodeFunctions createEncoderForLBL(int format, int codePage) {
- CodeFunctions funcs = new CodeFunctions();
-
- if (format == ENCODING_FORMAT6) {
- funcs.setEncodingType(ENCODING_FORMAT6);
- funcs.setEncoder(new Format6Encoder());
- funcs.setDecoder(new Format6Decoder());
- } else if (codePage == 932) {
- funcs.setEncodingType(ENCODING_FORMAT10);
- funcs.setEncoder(new AnyCharsetEncoder("ms932"));
- funcs.setDecoder(new AnyCharsetDecoder("ms932"));
- } else if (format == ENCODING_FORMAT9) {
- funcs.setEncodingType(ENCODING_FORMAT9);
- String cpName = "cp" + codePage;
- if (codePage == 1252)
- funcs.setEncoder(new LatinEncoder());
- else
- funcs.setEncoder(new AnyCharsetEncoder(cpName));
- guessCodepage(funcs, cpName);
-
- funcs.setDecoder(new AnyCharsetDecoder(cpName));
- } else {
- // TODO TEMP...
- funcs.setEncodingType(ENCODING_FORMAT9);
- funcs.setEncoder(new AnyCharsetEncoder("cp1252"));
- funcs.setDecoder(new AnyCharsetDecoder("cp1252"));
- }
-
- return funcs;
- }
-
public static CharacterEncoder getDefaultEncoder() {
return new Format6Encoder();
}
public static CharacterDecoder getDefaultDecoder() {
return new Format6Decoder();
- }
-
- public static Transliterator getDefaultTransliterator() {
- TableTransliterator ascii = new TableTransliterator("ascii");
- ascii.forceUppercase(true);
- return ascii;
}
}
Index: test/uk/me/parabola/imgfmt/app/labelenc/CodeFunctionsTest.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- test/uk/me/parabola/imgfmt/app/labelenc/CodeFunctionsTest.java (revision 2687)
+++ test/uk/me/parabola/imgfmt/app/labelenc/CodeFunctionsTest.java (revision )
@@ -58,7 +58,7 @@
CodeFunctions functions = CodeFunctions.createEncoderForLBL(6, 0);
CharacterEncoder encoder = functions.getEncoder();
- Transliterator transliterator = functions.getTransliterator();
+ Transliterator transliterator = new TableTransliterator("ascii");
EncodedText text = encoder.encodeText(transliterator.transliterate("KörnerstraÃe, VelkomezeÅÃÄská, Skólavörðustigur"));
CharacterDecoder decoder = functions.getDecoder();
@@ -72,15 +72,15 @@
}
/**
- * Transliteration when going to ascii in format 6. This was originally
- * the only place where transliteration was available.
- */
+ * Transliteration when going to ascii in format 6. This was originally
+ * the only place where transliteration was available.
+ */
@Test
public void testTransliterateLatin() {
CodeFunctions functions = CodeFunctions.createEncoderForLBL("latin1");
CharacterEncoder encoder = functions.getEncoder();
- Transliterator transliterator = functions.getTransliterator();
+ Transliterator transliterator = new TableTransliterator("latin1");
EncodedText text = encoder.encodeText(transliterator.transliterate("KörnerstraÃe, VelkomezeÅÃÄská, Skólavörðustigur"));
CharacterDecoder decoder = functions.getDecoder();
@@ -88,7 +88,7 @@
for (int i = 0; i < text.getLength(); i++) {
decoder.addByte(ctext[i]);
}
-
+
String result = decoder.getText().getText();
assertEquals("transliterated text", "KörnerstraÃe, VelkomezerÃcská, Skólavörðustigur", result);
}
Index: src/uk/me/parabola/imgfmt/app/labelenc/LatinEncoder.java
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/LatinEncoder.java (revision 2687)
+++ src/uk/me/parabola/imgfmt/app/labelenc/LatinEncoder.java (revision 2687)
@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2009.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 or
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-package uk.me.parabola.imgfmt.app.labelenc;
-
-import java.nio.charset.Charset;
-import java.util.Locale;
-
-/**
- * An encoder for latin script
- * @author Steve Ratcliffe
- */
-public class LatinEncoder extends BaseEncoder implements CharacterEncoder {
- private final Charset latinCharset = Charset.forName("latin1");
-
- public EncodedText encodeText(String t) {
- String text;
- if (t != null && isUpperCase())
- text = t.toUpperCase(Locale.ENGLISH);
- else
- text = t;
- // Need to add a null character at the end of the string for this format.
- String zText = text + "\000";
-
- byte[] chars = zText.getBytes(latinCharset);
- return new EncodedText(chars, chars.length);
- }
-}
Index: src/uk/me/parabola/imgfmt/app/labelenc/Format6Encoder.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- src/uk/me/parabola/imgfmt/app/labelenc/Format6Encoder.java (revision 2687)
+++ src/uk/me/parabola/imgfmt/app/labelenc/Format6Encoder.java (revision )
@@ -48,6 +48,8 @@
"xxxxxxxxxx:;<=>?" + // 0x10-0x1F
"xxxxxxxxxxx[\\]^_"; // 0x20-0x2F
+ private final Transliterator transliterator = new TableTransliterator("ascii");
+
/**
* Encode the text into the 6 bit format. See the class level notes.
*
@@ -56,10 +58,10 @@
* some escape sequences will be present.
*/
public EncodedText encodeText(String text) {
- if (text == null || text.length() == 0)
+ if (text == null || text.isEmpty())
return NO_TEXT;
- String s = text.toUpperCase(Locale.ENGLISH);
+ String s = transliterator.transliterate(text).toUpperCase(Locale.ENGLISH);
// Allocate more than enough space on average for the label.
// if you overdo it then it will waste a lot of space , but
Index: src/uk/me/parabola/imgfmt/app/lbl/LBLFile.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- src/uk/me/parabola/imgfmt/app/lbl/LBLFile.java (revision 2687)
+++ src/uk/me/parabola/imgfmt/app/lbl/LBLFile.java (revision )
@@ -28,7 +28,6 @@
import uk.me.parabola.imgfmt.app.labelenc.BaseEncoder;
import uk.me.parabola.imgfmt.app.labelenc.CharacterEncoder;
import uk.me.parabola.imgfmt.app.labelenc.CodeFunctions;
-import uk.me.parabola.imgfmt.app.labelenc.Transliterator;
import uk.me.parabola.imgfmt.app.srt.Sort;
import uk.me.parabola.imgfmt.app.trergn.Subdivision;
import uk.me.parabola.imgfmt.fs.ImgChannel;
@@ -48,7 +47,6 @@
private static final Logger log = Logger.getLogger(LBLFile.class);
private CharacterEncoder textEncoder = CodeFunctions.getDefaultEncoder();
- private Transliterator transliterator = CodeFunctions.getDefaultTransliterator();
private final Map<String, Label> labelCache = new HashMap<String, Label>();
@@ -102,13 +100,10 @@
lblHeader.setEncodingType(cfuncs.getEncodingType());
textEncoder = cfuncs.getEncoder();
- transliterator = cfuncs.getTransliterator();
if (forceUpper && textEncoder instanceof BaseEncoder) {
BaseEncoder baseEncoder = (BaseEncoder) textEncoder;
baseEncoder.setUpperCase(true);
}
- if (forceUpper)
- transliterator.forceUppercase(true);
}
public void setEncoder(int encodingType, int codepage ) {
@@ -116,18 +111,16 @@
lblHeader.setEncodingType(cfuncs.getEncodingType());
textEncoder = cfuncs.getEncoder();
- transliterator = cfuncs.getTransliterator();
}
/**
* Add a new label with the given text. Labels are shared, so that identical
* text is always represented by the same label.
*
- * @param inText The text of the label, it will be in uppercase.
+ * @param text The text of the label, it will be in uppercase.
* @return A reference to the created label.
*/
- public Label newLabel(String inText) {
- String text = transliterator.transliterate(inText);
+ public Label newLabel(String text) {
Label l = labelCache.get(text);
if (l == null) {
l = new Label(text);
_______________________________________________
mkgmap-dev mailing list
[email protected]
http://lists.mkgmap.org.uk/mailman/listinfo/mkgmap-dev
