Was doing some codepage work (playing with CP437), and found that the
Pike CP-437 decoder actually produces wrong output, according to the
unicode.org template file. I'm not sure what's going on here; is there
some alternate standard that it's adhering to?
Here are two scripts to compare codepage decoding in Pike and Python:
int main(int argc,array(string) argv)
{
object decoder=Charset.decoder(argv[1]);
for (int byte=128;byte<255;++byte)
write("%02x: %04x\n",byte,decoder->feed((string)({byte}))->drain()[0]);
}
import sys
for byte in range(128,255):
print("%02x:
%04x"%(byte,ord(bytes([byte]).decode(sys.argv[1],errors="replace"))))
$ diff <(pike charsets.pike 437) <(python3 charsets.py 437)
This should in theory be absolutely silent, but in practice it reports
a number of differences in the decode. Wikipedia cites a file on
www.unicode.org as its source:
http://en.wikipedia.org/wiki/Code_page_437
http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP437.TXT
and that file agrees with Python's decoding. I've put together a
script to download a file from the above URL and patch the info into
the appropriate place in src/modules/_Charset/misc.c, and have run it
on everything in the MICSFT/PC/ directory; the resulting patch is
attached, as is a patch to create that file.
ChrisA
From 86fdbc9d4e18651999a33f4fda235bb1ece8de49 Mon Sep 17 00:00:00 2001
From: Chris Angelico <ros...@gmail.com>
Date: Mon, 4 Aug 2014 12:00:17 +1000
Subject: [PATCH 1/2] Correct MS/IBM codepage decoding tables based on
www.unicode.org files
---
src/modules/_Charset/misc.c | 212 +++++++++++++++++++++----------------------
1 file changed, 106 insertions(+), 106 deletions(-)
diff --git a/src/modules/_Charset/misc.c b/src/modules/_Charset/misc.c
index 51ed656..f561645 100644
--- a/src/modules/_Charset/misc.c
+++ b/src/modules/_Charset/misc.c
@@ -1010,16 +1010,16 @@ static const p_wchar1 map_IBM437[] = {
0x00ff, 0x00d6, 0x00dc, 0x00a2, 0x00a3, 0x00a5, 0x20a7, 0x0192,
0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x00f1, 0x00d1, 0x00aa, 0x00ba,
0x00bf, 0x2310, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
- 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2525, 0x2528, 0x2512,
- 0x2511, 0x252b, 0x2503, 0x2513, 0x251b, 0x251a, 0x2519, 0x2510,
- 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x251d, 0x2520,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x2537,
- 0x2538, 0x252f, 0x2530, 0x2516, 0x2515, 0x250d, 0x250e, 0x2542,
- 0x253f, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
- 0x03b1, 0x03b2, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x03bc, 0x03c4,
- 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x2205, 0x03b5, 0x2229,
+ 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556,
+ 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510,
+ 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567,
+ 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b,
+ 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
+ 0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x00b5, 0x03c4,
+ 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x03c6, 0x03b5, 0x2229,
0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248,
- 0x2218, 0x00b7, 0x2219, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
+ 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM500[] = {
0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd,
0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd,
@@ -1057,15 +1057,15 @@ static const p_wchar1 map_IBM850[] = {
0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x00f1, 0x00d1, 0x00aa, 0x00ba,
0x00bf, 0x00ae, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00c1, 0x00c2, 0x00c0,
- 0x00a9, 0x252b, 0x2503, 0x2513, 0x251b, 0x00a2, 0x00a5, 0x2510,
+ 0x00a9, 0x2563, 0x2551, 0x2557, 0x255d, 0x00a2, 0x00a5, 0x2510,
0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x00e3, 0x00c3,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x00a4,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x00a4,
0x00f0, 0x00d0, 0x00ca, 0x00cb, 0x00c8, 0x0131, 0x00cd, 0x00ce,
0x00cf, 0x2518, 0x250c, 0x2588, 0x2584, 0x00a6, 0x00cc, 0x2580,
- 0x00d3, 0x00df, 0x00d4, 0x00d2, 0x00f5, 0x00d5, 0x03bc, 0x00de,
- 0x00fe, 0x00da, 0x00db, 0x00d9, 0x00fd, 0x00dd, 0x2014, 0x00b4,
- 0x00ad, 0x00b1, 0x21d4, 0x00be, 0x00b6, 0x00a7, 0x00f7, 0x02db,
- 0x00b0, 0x00a8, 0x02d9, 0x00b9, 0x00b3, 0x00b2, 0x25a0, 0x00a0, };
+ 0x00d3, 0x00df, 0x00d4, 0x00d2, 0x00f5, 0x00d5, 0x00b5, 0x00fe,
+ 0x00de, 0x00da, 0x00db, 0x00d9, 0x00fd, 0x00dd, 0x00af, 0x00b4,
+ 0x00ad, 0x00b1, 0x2017, 0x00be, 0x00b6, 0x00a7, 0x00f7, 0x00b8,
+ 0x00b0, 0x00a8, 0x00b7, 0x00b9, 0x00b3, 0x00b2, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM851[] = {
0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, 0x0386, 0x00e7,
0x00ea, 0x00eb, 0x00e8, 0x00ef, 0x00ee, 0x0388, 0x00c4, 0x0389,
@@ -1089,34 +1089,34 @@ static const p_wchar1 map_IBM852[] = {
0x00c9, 0x0139, 0x013a, 0x00f4, 0x00f6, 0x013d, 0x013e, 0x015a,
0x015b, 0x00d6, 0x00dc, 0x0164, 0x0165, 0x0141, 0x00d7, 0x010d,
0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x0104, 0x0105, 0x017d, 0x017e,
- 0x0118, 0x0119, 0xfffd, 0x017a, 0x010c, 0x015f, 0x00ab, 0x00bb,
+ 0x0118, 0x0119, 0x00ac, 0x017a, 0x010c, 0x015f, 0x00ab, 0x00bb,
0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00c1, 0x00c2, 0x011a,
- 0x015e, 0x252b, 0x2503, 0x2513, 0x251b, 0x017b, 0x017c, 0x2510,
+ 0x015e, 0x2563, 0x2551, 0x2557, 0x255d, 0x017b, 0x017c, 0x2510,
0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x0102, 0x0103,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x00a4,
- 0x0111, 0x0110, 0x010e, 0x00cb, 0x010f, 0x0147, 0x00cd, 0x00db,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x00a4,
+ 0x0111, 0x0110, 0x010e, 0x00cb, 0x010f, 0x0147, 0x00cd, 0x00ce,
0x011b, 0x2518, 0x250c, 0x2588, 0x2584, 0x0162, 0x016e, 0x2580,
0x00d3, 0x00df, 0x00d4, 0x0143, 0x0144, 0x0148, 0x0160, 0x0161,
0x0154, 0x00da, 0x0155, 0x0170, 0x00fd, 0x00dd, 0x0163, 0x00b4,
- 0x00ad, 0x02dd, 0x00b8, 0x02c7, 0x02d8, 0x00a7, 0x00f7, 0x02db,
+ 0x00ad, 0x02dd, 0x02db, 0x02c7, 0x02d8, 0x00a7, 0x00f7, 0x00b8,
0x00b0, 0x00a8, 0x02d9, 0x0171, 0x0158, 0x0159, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM855[] = {
0x0452, 0x0402, 0x0453, 0x0403, 0x0451, 0x0401, 0x0454, 0x0404,
0x0455, 0x0405, 0x0456, 0x0406, 0x0457, 0x0407, 0x0458, 0x0408,
- 0x0459, 0x0409, 0x045a, 0x040a, 0x045b, 0x0093, 0x045c, 0x040c,
+ 0x0459, 0x0409, 0x045a, 0x040a, 0x045b, 0x040b, 0x045c, 0x040c,
0x045e, 0x040e, 0x045f, 0x040f, 0x044e, 0x042e, 0x044a, 0x042a,
0x0430, 0x0410, 0x0431, 0x0411, 0x0446, 0x0426, 0x0434, 0x0414,
0x0435, 0x0415, 0x0444, 0x0424, 0x0433, 0x0413, 0x00ab, 0x00bb,
0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x0445, 0x0425, 0x0438,
- 0x0418, 0x252b, 0x2503, 0x2513, 0x251b, 0x0439, 0x0419, 0x2510,
+ 0x0418, 0x2563, 0x2551, 0x2557, 0x255d, 0x0439, 0x0419, 0x2510,
0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x043a, 0x041a,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x00a4,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x00a4,
0x043b, 0x041b, 0x043c, 0x041c, 0x043d, 0x041d, 0x043e, 0x041e,
0x043f, 0x2518, 0x250c, 0x2588, 0x2584, 0x041f, 0x044f, 0x2580,
0x042f, 0x0440, 0x0420, 0x0441, 0x0421, 0x0442, 0x0422, 0x0443,
- 0x0423, 0x0436, 0x0416, 0x0432, 0x0412, 0x044c, 0x042c, 0x00b4,
+ 0x0423, 0x0436, 0x0416, 0x0432, 0x0412, 0x044c, 0x042c, 0x2116,
0x00ad, 0x044b, 0x042b, 0x0437, 0x0417, 0x0448, 0x0428, 0x044d,
- 0x042d, 0x0449, 0x0429, 0x0447, 0x0427, 0xfffd, 0x25a0, 0x00a0, };
+ 0x042d, 0x0449, 0x0429, 0x0447, 0x0427, 0x00a7, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM857[] = {
0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, 0x00e5, 0x00e7,
0x00ea, 0x00eb, 0x00e8, 0x00ef, 0x00ee, 0x0131, 0x00c4, 0x00c5,
@@ -1125,117 +1125,117 @@ static const p_wchar1 map_IBM857[] = {
0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x00f1, 0x00d1, 0x011e, 0x011f,
0x00bf, 0x00ae, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00c1, 0x00c2, 0x00c0,
- 0x00a9, 0x252b, 0x2503, 0x2513, 0x251b, 0x00a2, 0x00a5, 0x2510,
+ 0x00a9, 0x2563, 0x2551, 0x2557, 0x255d, 0x00a2, 0x00a5, 0x2510,
0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x00e3, 0x00c3,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x00a4,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x00a4,
0x00ba, 0x00aa, 0x00ca, 0x00cb, 0x00c8, 0xfffd, 0x00cd, 0x00ce,
0x00cf, 0x2518, 0x250c, 0x2588, 0x2584, 0x00a6, 0x00cc, 0x2580,
- 0x00d3, 0x00df, 0x00d4, 0x00d2, 0x00f5, 0x00d5, 0x03bc, 0xfffd,
- 0x00d7, 0x00da, 0x00db, 0x00d9, 0x00ec, 0x00ff, 0x2014, 0x00b4,
- 0x00ad, 0x00b1, 0xfffd, 0x00be, 0x00b6, 0x00a7, 0x00f7, 0x02db,
- 0x00b0, 0x00a8, 0x02d9, 0x00b9, 0x00b3, 0x00b2, 0x25a0, 0x00a0, };
+ 0x00d3, 0x00df, 0x00d4, 0x00d2, 0x00f5, 0x00d5, 0x00b5, 0xfffd,
+ 0x00d7, 0x00da, 0x00db, 0x00d9, 0x00ec, 0x00ff, 0x00af, 0x00b4,
+ 0x00ad, 0x00b1, 0xfffd, 0x00be, 0x00b6, 0x00a7, 0x00f7, 0x00b8,
+ 0x00b0, 0x00a8, 0x00b7, 0x00b9, 0x00b3, 0x00b2, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM860[] = {
0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e3, 0x00e0, 0x00c1, 0x00e7,
- 0x00ea, 0x00ca, 0x00e8, 0x00ce, 0x00d4, 0x00ec, 0x00c3, 0x00c2,
+ 0x00ea, 0x00ca, 0x00e8, 0x00cd, 0x00d4, 0x00ec, 0x00c3, 0x00c2,
0x00c9, 0x00c0, 0x00c8, 0x00f4, 0x00f5, 0x00f2, 0x00da, 0x00f9,
- 0x00cc, 0x00d5, 0x00dc, 0x00a2, 0x00a3, 0x00d9, 0x20a7, 0x00d2,
+ 0x00cc, 0x00d5, 0x00dc, 0x00a2, 0x00a3, 0x00d9, 0x20a7, 0x00d3,
0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x00f1, 0x00d1, 0x00aa, 0x00ba,
- 0x00bf, 0x00d3, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
- 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2525, 0x2528, 0x2512,
- 0x2511, 0x252b, 0x2503, 0x2513, 0x251b, 0x251a, 0x2519, 0x2510,
- 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x251d, 0x2520,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x2537,
- 0x2538, 0x252f, 0x2530, 0x2516, 0x2515, 0x250d, 0x250e, 0x2542,
- 0x253f, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
- 0x03b1, 0x03b2, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x03bc, 0x03c4,
- 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x2205, 0x03b5, 0x2229,
+ 0x00bf, 0x00d2, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
+ 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556,
+ 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510,
+ 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567,
+ 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b,
+ 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
+ 0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x00b5, 0x03c4,
+ 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x03c6, 0x03b5, 0x2229,
0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248,
- 0x2218, 0x00b7, 0x2219, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
+ 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM861[] = {
0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, 0x00e5, 0x00e7,
0x00ea, 0x00eb, 0x00e8, 0x00d0, 0x00f0, 0x00de, 0x00c4, 0x00c5,
0x00c9, 0x00e6, 0x00c6, 0x00f4, 0x00f6, 0x00fe, 0x00fb, 0x00dd,
- 0x00fd, 0x00d6, 0x00dc, 0x00f8, 0x00a3, 0x00d8, 0x20a7, 0x00d2,
+ 0x00fd, 0x00d6, 0x00dc, 0x00f8, 0x00a3, 0x00d8, 0x20a7, 0x0192,
0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x00c1, 0x00cd, 0x00d3, 0x00da,
- 0x00bf, 0xfffd, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
- 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2525, 0x2528, 0x2512,
- 0x2511, 0x252b, 0x2503, 0x2513, 0x251b, 0x251a, 0x2519, 0x2510,
- 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x251d, 0x2520,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x2537,
- 0x2538, 0x252f, 0x2530, 0x2516, 0x2515, 0x250d, 0x250e, 0x2542,
- 0x253f, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
- 0x03b1, 0x03b2, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x03bc, 0x03c4,
- 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x2205, 0x03b5, 0x2229,
+ 0x00bf, 0x2310, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
+ 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556,
+ 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510,
+ 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567,
+ 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b,
+ 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
+ 0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x00b5, 0x03c4,
+ 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x03c6, 0x03b5, 0x2229,
0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248,
- 0x2218, 0x00b7, 0x2219, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
+ 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM862[] = {
0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7,
0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df,
0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7,
- 0x05e8, 0x05e9, 0x05ea, 0x00a2, 0x00a3, 0x00d9, 0x20a7, 0x00d2,
+ 0x05e8, 0x05e9, 0x05ea, 0x00a2, 0x00a3, 0x00a5, 0x20a7, 0x0192,
0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x00f1, 0x00d1, 0x00aa, 0x00ba,
- 0x00bf, 0xfffd, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
- 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2525, 0x2528, 0x2512,
- 0x2511, 0x252b, 0x2503, 0x2513, 0x251b, 0x251a, 0x2519, 0x2510,
- 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x251d, 0x2520,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x2537,
- 0x2538, 0x252f, 0x2530, 0x2516, 0x2515, 0x250d, 0x250e, 0x2542,
- 0x253f, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
- 0x03b1, 0x03b2, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x03bc, 0x03c4,
- 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x2205, 0x03b5, 0x2229,
+ 0x00bf, 0x2310, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
+ 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556,
+ 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510,
+ 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567,
+ 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b,
+ 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
+ 0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x00b5, 0x03c4,
+ 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x03c6, 0x03b5, 0x2229,
0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248,
- 0x2218, 0x00b7, 0x2219, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
+ 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM863[] = {
0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00c2, 0x00e0, 0x00b6, 0x00e7,
- 0x00ea, 0x00eb, 0x00e8, 0x00ef, 0x00ee, 0x00ec, 0x00c0, 0x00a7,
+ 0x00ea, 0x00eb, 0x00e8, 0x00ef, 0x00ee, 0x2017, 0x00c0, 0x00a7,
0x00c9, 0x00c8, 0x00ca, 0x00f4, 0x00cb, 0x00cf, 0x00fb, 0x00f9,
0x00a4, 0x00d4, 0x00dc, 0x00a2, 0x00a3, 0x00d9, 0x00db, 0x0192,
- 0x00a6, 0x00b4, 0x00f3, 0x00fa, 0x00a8, 0x00b8, 0x00b3, 0x203e,
+ 0x00a6, 0x00b4, 0x00f3, 0x00fa, 0x00a8, 0x00b8, 0x00b3, 0x00af,
0x00ce, 0x2310, 0x00ac, 0x00bd, 0x00bc, 0x00be, 0x00ab, 0x00bb,
- 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2525, 0x2528, 0x2512,
- 0x2511, 0x252b, 0x2503, 0x2513, 0x251b, 0x251a, 0x2519, 0x2510,
- 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x251d, 0x2520,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x2537,
- 0x2538, 0x252f, 0x2530, 0x2516, 0x2515, 0x250d, 0x250e, 0x2542,
- 0x253f, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
- 0x03b1, 0x03b2, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x03bc, 0x03c4,
- 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x2205, 0x03b5, 0x2229,
+ 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556,
+ 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510,
+ 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567,
+ 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b,
+ 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
+ 0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x00b5, 0x03c4,
+ 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x03c6, 0x03b5, 0x2229,
0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248,
- 0x2218, 0x00b7, 0x2219, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
+ 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM864[] = {
- 0x00b0, 0x00b7, 0x2218, 0x221a, 0x2592, 0x2500, 0x2502, 0x253c,
+ 0x00b0, 0x00b7, 0x2219, 0x221a, 0x2592, 0x2500, 0x2502, 0x253c,
0x2524, 0x252c, 0x251c, 0x2534, 0x2510, 0x250c, 0x2514, 0x2518,
- 0x00df, 0x221e, 0x00f8, 0x00b1, 0x00bd, 0x00bc, 0x2248, 0x00ab,
- 0x00bb, 0xfef7, 0xfef8, 0xfffd, 0xfffd, 0xfefb, 0xfefc, 0xfe8e,
- 0xfffd, 0x00ad, 0xfe82, 0x00a3, 0x00a4, 0xfe84, 0xfffd, 0xfffd,
- 0xfe8e, 0x0628, 0x062a, 0x062b, 0x060c, 0x062c, 0x062d, 0x062e,
- 0x06f0, 0x06f1, 0x06f2, 0x06f3, 0x06f4, 0x06f5, 0x06f6, 0x06f7,
- 0x06f8, 0x06f9, 0x06a4, 0x061b, 0x0633, 0x0634, 0x0635, 0x061f,
- 0x00a2, 0x0621, 0x0622, 0x0623, 0x0624, 0xfecc, 0x0626, 0x0627,
- 0xfe90, 0x0629, 0xfe96, 0xfe9a, 0xfe9e, 0xfea2, 0xfea6, 0x062f,
- 0x0630, 0x0631, 0x0632, 0xfeb2, 0xfeb6, 0xfeba, 0xfebe, 0x0637,
- 0x0638, 0xfeca, 0xfece, 0x00a6, 0x00ac, 0x00f7, 0x00d7, 0x0639,
- 0x0640, 0xfed4, 0xfed8, 0xfeda, 0xfee0, 0xfee2, 0xfee8, 0xfeea,
- 0x0648, 0x0649, 0xfef2, 0x0636, 0xfee4, 0xfed0, 0x063a, 0x0645,
- 0xfe7d, 0x0651, 0x0646, 0x0647, 0xfeeb, 0xfef0, 0xfef4, 0x0641,
- 0x0642, 0xfef5, 0xfef6, 0x0644, 0x0643, 0x064a, 0x25a0, 0x00a0, };
+ 0x03b2, 0x221e, 0x03c6, 0x00b1, 0x00bd, 0x00bc, 0x2248, 0x00ab,
+ 0x00bb, 0xfef7, 0xfef8, 0xfffd, 0xfffd, 0xfefb, 0xfefc, 0xfffd,
+ 0x00a0, 0x00ad, 0xfe82, 0x00a3, 0x00a4, 0xfe84, 0xfffd, 0xfffd,
+ 0xfe8e, 0xfe8f, 0xfe95, 0xfe99, 0x060c, 0xfe9d, 0xfea1, 0xfea5,
+ 0x0660, 0x0661, 0x0662, 0x0663, 0x0664, 0x0665, 0x0666, 0x0667,
+ 0x0668, 0x0669, 0xfed1, 0x061b, 0xfeb1, 0xfeb5, 0xfeb9, 0x061f,
+ 0x00a2, 0xfe80, 0xfe81, 0xfe83, 0xfe85, 0xfeca, 0xfe8b, 0xfe8d,
+ 0xfe91, 0xfe93, 0xfe97, 0xfe9b, 0xfe9f, 0xfea3, 0xfea7, 0xfea9,
+ 0xfeab, 0xfead, 0xfeaf, 0xfeb3, 0xfeb7, 0xfebb, 0xfebf, 0xfec1,
+ 0xfec5, 0xfecb, 0xfecf, 0x00a6, 0x00ac, 0x00f7, 0x00d7, 0xfec9,
+ 0x0640, 0xfed3, 0xfed7, 0xfedb, 0xfedf, 0xfee3, 0xfee7, 0xfeeb,
+ 0xfeed, 0xfeef, 0xfef3, 0xfebd, 0xfecc, 0xfece, 0xfecd, 0xfee1,
+ 0xfe7d, 0x0651, 0xfee5, 0xfee9, 0xfeec, 0xfef0, 0xfef2, 0xfed0,
+ 0xfed5, 0xfef5, 0xfef6, 0xfedd, 0xfed9, 0xfef1, 0x25a0, 0xfffd, };
static const p_wchar1 map_IBM865[] = {
0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, 0x00e5, 0x00e7,
0x00ea, 0x00eb, 0x00e8, 0x00ef, 0x00ee, 0x00ec, 0x00c4, 0x00c5,
0x00c9, 0x00e6, 0x00c6, 0x00f4, 0x00f6, 0x00f2, 0x00fb, 0x00f9,
0x00ff, 0x00d6, 0x00dc, 0x00f8, 0x00a3, 0x00d8, 0x20a7, 0x0192,
0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x00f1, 0x00d1, 0x00aa, 0x00ba,
- 0x00bf, 0x2310, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
- 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2525, 0x2528, 0x2512,
- 0x2511, 0x252b, 0x2503, 0x2513, 0x251b, 0x251a, 0x2519, 0x2510,
- 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x251d, 0x2520,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x2537,
- 0x2538, 0x252f, 0x2530, 0x2516, 0x2515, 0x250d, 0x250e, 0x2542,
- 0x253f, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
- 0x03b1, 0x03b2, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x03bc, 0x03c4,
- 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x2205, 0x03b5, 0x2229,
+ 0x00bf, 0x2310, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00a4,
+ 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556,
+ 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510,
+ 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567,
+ 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b,
+ 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
+ 0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x00b5, 0x03c4,
+ 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x03c6, 0x03b5, 0x2229,
0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248,
- 0x2218, 0x00b7, 0x2219, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
+ 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM866 [] = {
0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417,
0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
@@ -1272,20 +1272,20 @@ static const p_wchar1 map_IBM868[] = {
0xfffd, 0xfffd, 0xfffd, 0x0651, 0xfe7d, 0xfffd, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM869[] = {
0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x0386, 0xfffd,
- 0x00b7, 0x00ac, 0x00a6, 0x201b, 0x2019, 0x0388, 0x2014, 0x0389,
+ 0x00b7, 0x00ac, 0x00a6, 0x2018, 0x2019, 0x0388, 0x2015, 0x0389,
0x038a, 0x03aa, 0x038c, 0xfffd, 0xfffd, 0x038e, 0x03ab, 0x00a9,
0x038f, 0x00b2, 0x00b3, 0x03ac, 0x00a3, 0x03ad, 0x03ae, 0x03af,
0x03ca, 0x0390, 0x03cc, 0x03cd, 0x0391, 0x0392, 0x0393, 0x0394,
0x0395, 0x0396, 0x0397, 0x00bd, 0x0398, 0x0399, 0x00ab, 0x00bb,
0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x039a, 0x039b, 0x039c,
- 0x039d, 0x252b, 0x2503, 0x2513, 0x251b, 0x039e, 0x039f, 0x2510,
+ 0x039d, 0x2563, 0x2551, 0x2557, 0x255d, 0x039e, 0x039f, 0x2510,
0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x03a0, 0x03a1,
- 0x2517, 0x250f, 0x253b, 0x2533, 0x2523, 0x2501, 0x254b, 0x03a3,
+ 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x03a3,
0x03a4, 0x03a5, 0x03a6, 0x03a7, 0x03a8, 0x03a9, 0x03b1, 0x03b2,
0x03b3, 0x2518, 0x250c, 0x2588, 0x2584, 0x03b4, 0x03b5, 0x2580,
0x03b6, 0x03b7, 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd,
- 0x03be, 0x03bf, 0x03c0, 0x03c1, 0x03c3, 0x03c2, 0x03c4, 0x00b4,
- 0x00ad, 0x00b1, 0x03c5, 0x03c6, 0x03c7, 0x00a7, 0x03c8, 0x03f4,
+ 0x03be, 0x03bf, 0x03c0, 0x03c1, 0x03c3, 0x03c2, 0x03c4, 0x0384,
+ 0x00ad, 0x00b1, 0x03c5, 0x03c6, 0x03c7, 0x00a7, 0x03c8, 0x0385,
0x00b0, 0x00a8, 0x03c9, 0x03cb, 0x03b0, 0x03ce, 0x25a0, 0x00a0, };
static const p_wchar1 map_IBM870[] = {
0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd,
--
1.7.10.4
From 59b321792541a3f137d1c9d5031fd3fe721634e9 Mon Sep 17 00:00:00 2001
From: Chris Angelico <ros...@gmail.com>
Date: Mon, 4 Aug 2014 12:13:06 +1000
Subject: [PATCH 2/2] Add the script used to make the codepage changes
---
src/modules/_Charset/fix_charset_decode.pike | 31 ++++++++++++++++++++++++++
1 file changed, 31 insertions(+)
create mode 100644 src/modules/_Charset/fix_charset_decode.pike
diff --git a/src/modules/_Charset/fix_charset_decode.pike b/src/modules/_Charset/fix_charset_decode.pike
new file mode 100644
index 0000000..200dc12
--- /dev/null
+++ b/src/modules/_Charset/fix_charset_decode.pike
@@ -0,0 +1,31 @@
+/*
+Download a codepage definition file from www.unicode.org and update misc.c
+accordingly. Assumes/expects that the codepage will have the first 128 bytes
+correspond to the ASCII characters, and renders only the next 128. Could
+probably be adapted to other codepages, but currently has a fixed URL for
+downloading the template files.
+*/
+int main(int argc,array(string) argv)
+{
+ if (argc<2) exit(1,"USAGE: pike %s codepage\nwhere codepage is an integer eg 437",argv[0]);
+ int cp=(int)argv[1];
+ //TODO: Find templates in other directories
+ string url=sprintf("http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP%d.TXT",cp);
+ string template=Protocols.HTTP.get_url_data(url);
+ if (!template) exit(1,"Code page %d not found - URL %s\n",cp,url);
+ string pattern="%sstatic const p_wchar1 map_IBM"+cp+"[] = {%s };%s";
+ sscanf(Stdio.read_file("misc.c"),pattern,string before,string current,string after);
+ if (!after) exit(0,"Code page %d not in use\n",cp);
+ array(int) output=allocate(128,0xfffd);
+ //They ought to be in order, but parse properly just in case
+ foreach (template/"\n",string line) if (sscanf(line,"0x%2x\t0x%4x\t",int byte,int char))
+ {
+ if (byte<128) continue; //assert(byte==char)
+ output[byte-128]=char||0xfffd; //If the replacement is blank, use U+FFFD
+ }
+ //Nesting %{ %} doesn't seem to work properly (??)
+ string newdata=sprintf("\n %{ 0x%04x,%}",(output/8)[*])*"";
+ if (newdata==current) exit(0,"Nothing to change for codepage %d\n",cp);
+ Stdio.write_file("misc.c",sprintf(pattern,before,newdata,after));
+ exit(0,"Updated data for codepage %d\n",cp);
+}
--
1.7.10.4
