Package: lookup Version: 1.08b-10 Severity: wishlist Tags: patch Attached is a patch to add utf-8 output support to lookup. In order to avoid an iconv dependancy, I've instead generated two arrays (JIS X 208 and JIS X 212) with a perl script.
However, input support without iconv looks painful, so I thought I'd submit
this as it is for consideration.
If you want to include the lib/jisucs2tbl.h in the tarball (like commands.h)
then simply generate it once, and move lib/jisucs2tbl.h from the clean target
to the realclean targe.
Adding any more output encodings would be difficult, as we just used up all the
input encoding bitfields.
diff -ruN lookup-1.08b.orig/cmds.master lookup-1.08b/cmds.master
--- lookup-1.08b.orig/cmds.master 1996-07-19 01:10:36.000000000 +1000
+++ lookup-1.08b/cmds.master 2005-09-01 18:36:22.000000000 +1000
@@ -322,8 +322,8 @@
CMD_GENERAL|CMD_ENCODING_RELATED
report or set the output encoding-method
-output encoding [euc|sjis|jis|...]
-output> (encoding>)?
(euc|sjis|jis-?(78|83|90)?(-(ascii|roman))?)?((<(212|no212|hwk|nohwk|foldhwk|disp|nodisp|code|mark)>|[-,\s]+)*)
+output encoding [euc|sjis|utf8|jis|...]
+output> (encoding>)?
(euc|sjis|utf8|jis-?(78|83|90)?(-(ascii|roman))?)?((<(212|no212|hwk|nohwk|foldhwk|disp|nodisp|code|mark)>|[-,\s]+)*)
cmd_output_encoding(\2, \3, \5, \6)
CMD_GENERAL
diff -ruN lookup-1.08b.orig/commands.c lookup-1.08b/commands.c
--- lookup-1.08b.orig/commands.c 2005-09-01 18:48:12.000000000 +1000
+++ lookup-1.08b/commands.c 2005-09-01 18:32:14.000000000 +1000
@@ -1367,6 +1367,7 @@
default: soft_assert(0); break;
case SJIS_OUTPUT: output("sjis"); break;
case EUC_OUTPUT: output("euc"); break;
+ case UTF8_OUTPUT: output("utf8"); break;
case JIS_OUTPUT:
switch(output_style & _JIS_KANJI_STYLE)
@@ -1424,6 +1425,8 @@
(void)select_output_style(EUC_OUTPUT);
else if (main_style[0]=='s' || main_style[0]=='S')
(void)select_output_style(SJIS_OUTPUT);
+ else if (main_style[0]=='u' || main_style[0]=='U')
+ (void)select_output_style(UTF8_OUTPUT);
else if (main_style[0]=='j' || main_style[0]=='j')
{
if (!jis_year)
diff -ruN lookup-1.08b.orig/commands.h lookup-1.08b/commands.h
--- lookup-1.08b.orig/commands.h 1996-07-21 18:52:23.000000000 +1000
+++ lookup-1.08b/commands.h 2005-09-01 18:36:43.000000000 +1000
@@ -696,9 +696,9 @@
/* generated from "cmds.master" record at line 320*/
{
CMD_GENERAL|CMD_ENCODING_RELATED,
- (S)"output encoding [euc|sjis|jis|...]",
+ (S)"output encoding [euc|sjis|utf8|jis|...]",
(S)"report or set the output encoding-method",
-
(S)"^\\s*output>\\s*(encoding>)?\\s*(euc|sjis|jis-?(78|83|90)?(-(ascii|roman))?)?((<(212|no212|hwk|nohwk|foldhwk|disp|nodisp|code|mark)>|[-,\\s]+)*)\\s*$",
+
(S)"^\\s*output>\\s*(encoding>)?\\s*(euc|sjis|utf8|jis-?(78|83|90)?(-(ascii|roman))?)?((<(212|no212|hwk|nohwk|foldhwk|disp|nodisp|code|mark)>|[-,\\s]+)*)\\s*$",
_func41_,
},
diff -ruN lookup-1.08b.orig/lib/output.c lookup-1.08b/lib/output.c
--- lookup-1.08b.orig/lib/output.c 2005-09-01 18:48:12.000000000 +1000
+++ lookup-1.08b/lib/output.c 2005-09-01 18:35:27.000000000 +1000
@@ -509,6 +509,110 @@
return retval;
}
+#include "jisucs2tbl.h"
+
+/* We take the next character to print, and return the number
+ * of columns we've output. Bytes go to *output++
+ */
+static unsigned output_euc_as_utf8(unsigned char c)
+{
+ static unsigned char hi = 0;
+ static unsigned char mid = 0;
+ u_int16_t ucs2;
+ int width;
+
+ // EUC-JP can be one, two or three bytes
+ // First byte:
+ // 0x8e Halfwidth Kana (JIS X 0201 Kana) - 2 bytes HALF_WIDTH_KATA_HI
+ // Easy! UFF{Second byte - 0x40}
+ // 0x8f JIS X 0212 - 3 bytes THREE_BYTE_HI
+ // > 0xa0 JIS X 0208 - 2 bytes - Mapping table
+ // Otherwise, ASCII (JIS X 0201 Roman)
+
+// fprintf(stderr, "0x%02x (0x%02x) ", c, hi);
+ /* Catch incomplete characters */
+ switch (hi) {
+ case 0:
+ // New character time
+ if (c < 0xa0 && c != HALF_WIDTH_KATA_HI && c !=
THREE_BYTE_HI) {
+ // JIS X 0201 Roman
+ hi = 0;
+ mid = 0;
+ if (c != 0)
+ *nextout++ = c;
+ if ((c == '\n' && flush_on_newline) || nextout
>= bufend)
+ flush_raw_output();
+ // I'm sure there're better definitions of
printable than this...
+ return (c >= ' ') ? 1:0;
+ } else {
+ // Worry about it later
+ hi = c;
+ mid = 0;
+ return 0;
+ }
+ break;
+ case THREE_BYTE_HI:
+ // JIS X 0212
+ if (mid == 0) {
+ mid = c;
+ return 0;
+ }
+ break;
+ }
+
+ /* Completed multibyte characters */
+ switch(hi) {
+ case HALF_WIDTH_KATA_HI:
+ // JIS X 0201 Kana
+ ucs2 = 0xff00 + c - 0x40;
+ hi = 0;
+ mid = 0;
+ width = 1;
+ break;
+
+ case THREE_BYTE_HI:
+ // JIS X 0212
+ ucs2 = jis212[mid-0xa0][c-0xa0];
+ hi = 0;
+ mid = 0;
+ width = 2;
+ break;
+
+ default:
+ // JIS X 0208
+ ucs2 = jis208[hi-0xa0][c-0xa0];
+ hi = 0;
+ mid = 0;
+ width = 2;
+ break;
+ }
+
+ if (ucs2 < 0x80) {
+ *nextout++ = ucs2;
+ // If this happens, we've been translated down to ASCII.
+ if ((c == '\n' && flush_on_newline) || nextout >= bufend)
+ flush_raw_output();
+ return 1;
+ } else if (ucs2 < 0x800) {
+ *nextout++ = (0xc0 | (ucs2 >> 6));
+ *nextout++ = (0x80 | (ucs2 & 0x3f));
+ // If this happens, we're prolly back in single-width land
+ width = 1;
+ } else /* if (ucs2 < 0x10000)*/ {
+ *nextout++ = (0xe0 | (ucs2 >> 12));
+ *nextout++ = (0x80 | ((ucs2 >> 6) & 0x3f));
+ *nextout++ = (0x80 | (ucs2 & 0x3f));
+ /*
+ } else {
+ // Should never get here, EUC-JP doens't translate outside BMP.
+ return 0;
+ */
+ }
+ if (nextout >= bufend)
+ flush_raw_output();
+ return width;
+}
+
/***************************************************************/
unsigned (*_output_char_function)(unsigned char) = output_euc_as_jis;
@@ -986,6 +1090,9 @@
break;
}
break;
+ case UTF8_OUTPUT:
+ *function_pointer = output_euc_as_utf8;
+ break;
}
if ((output_style & _KATAKANA) != PASS_HW_KATANANA)
@@ -1019,6 +1126,7 @@
case JIS_ASCII: output(", ASCII)"); break;
}
break;
+ case UTF8_OUTPUT: output("UTF-8"); break;
}
switch (output_style & _0212_1990)
diff -ruN lookup-1.08b.orig/lib/output.h lookup-1.08b/lib/output.h
--- lookup-1.08b.orig/lib/output.h 1996-01-16 04:50:39.000000000 +1100
+++ lookup-1.08b/lib/output.h 2005-09-01 13:48:34.000000000 +1000
@@ -61,7 +61,8 @@
#define EUC_OUTPUT 0x00000001
#define SJIS_OUTPUT 0x00000002
#define JIS_OUTPUT 0x00000004
-#define _BASIC_OUTPUT_TYPE (JIS_OUTPUT|SJIS_OUTPUT|EUC_OUTPUT)
+#define UTF8_OUTPUT 0x00000008
+#define _BASIC_OUTPUT_TYPE (JIS_OUTPUT|SJIS_OUTPUT|EUC_OUTPUT|UTF8_OUTPUT)
#define JIS_1978_OUTPUT JIS_OUTPUT
#define JIS_1983_OUTPUT JIS_OUTPUT|0x00000010
diff -ruN lookup-1.08b.orig/Makefile lookup-1.08b/Makefile
--- lookup-1.08b.orig/Makefile 2005-09-01 18:48:12.000000000 +1000
+++ lookup-1.08b/Makefile 2005-09-01 18:56:15.000000000 +1000
@@ -175,6 +175,9 @@
-echo '#endif /* file wrapper */' >> tmp;
mv tmp lib/system.h
+lib/jisucs2tbl.h: /usr/share/i18n/charmaps/EUC-JP.gz
+ zcat /usr/share/i18n/charmaps/EUC-JP.gz | ./mkeucucs2tbl.pl >
lib/jisucs2tbl.h
+
make.sh: realclean
@echo ':# script to make lookup' > tmp
@echo '## Can set CC= and CFLAGS= on the command line, just as with
make' >> tmp
@@ -219,7 +222,7 @@
clean: tidy
[EMAIL PROTECTED] dummy > dummy.o
- /bin/rm -f \#* *.o *.d doc/#* $(LOCAL_LIB) lib/system.h
+ /bin/rm -f \#* *.o *.d doc/#* $(LOCAL_LIB) lib/system.h lib/jisucs2tbl.h
realclean: clean
[EMAIL PROTECTED] dummy > lookup.man.xxx
@@ -261,7 +264,7 @@
lib/virtfile.h lib/romaji2kana.h lib/jregex.h lib/strsave.h \
lib/replace.h lib/input.h lookup.h lib/jreadline.h
output.o: lib/output.c lib/config.h lib/assert.h lib/input.h \
- lib/output.h
+ lib/output.h lib/jisucs2tbl.h
replace.o: lib/replace.c lib/config.h lib/assert.h lib/jregex.h \
lib/xmalloc.h lib/replace.h
romaji2kana.o: lib/romaji2kana.c lib/config.h lib/assert.h \
diff -ruN lookup-1.08b.orig/mkeucucs2tbl.pl lookup-1.08b/mkeucucs2tbl.pl
--- lookup-1.08b.orig/mkeucucs2tbl.pl 1970-01-01 10:00:00.000000000 +1000
+++ lookup-1.08b/mkeucucs2tbl.pl 2005-09-01 18:25:32.000000000 +1000
@@ -0,0 +1,63 @@
+#! /usr/bin/perl
+
+# Handy reference: http://czyborra.com/utf/
+# Expects zcat /usr/share/i18n/charmaps/EUC-JP.gz as input
+
+while ( <> ) {
+ last if /JIS X 0208/;
+}
+
+my @jis208;
+while ( <> ) {
+ last if /JIS X 0212/;
+ next unless m#^<U(....)>\s+/x(..)/x(..)#;
+ my ($ucs, $row, $col) = (hex($1), hex($2)-0xa0, hex($3)-0xa0);
+ $jis208[$row][$col] = $ucs;
+# print "Character 0x$ucs is at JIS X 0208 row $row, col $col\n";
+}
+
+my @jis212;
+while ( <> ) {
+ next unless m#^<U(....)>\s+/x8f/x(..)/x(..)#;
+ my ($ucs, $row, $col) = (hex($1), hex($2)-0xa0, hex($3)-0xa0);
+ $jis212[$row][$col] = $ucs;
+# print "Character 0x$ucs is at JIS X 0212 row $row, col $col\n";
+}
+
+print "// JIS X 208: ".scalar(@jis208)." rows\n";
+
+print "static const u_int32_t jis208[", 0xfe - 0xa0 + 1 ,"][", 0xfe-0xa0 + 1
,"] = {\n";
+for my $i (0 .. 0xfe-0xa0) {
+ print "\t{";
+ for my $j (0 .. 0xfe-0xa0) {
+ if (defined $jis208[$i][$j]) {
+ printf "0x%04x", $jis208[$i][$j];
+ } else {
+ print "0x0000";
+ }
+ print ", " unless $j == 0xfe-0xa0;
+ }
+ print "}";
+ print ", " unless $i == 0xfe-0xa0;
+ print "\n";
+}
+print "};\n";
+
+print "// JIS X 212: ".scalar(@jis212)." rows\n";
+
+print "static const u_int32_t jis212[", 0xfe - 0xa0 + 1 ,"][", 0xfe-0xa0 + 1
,"] = {\n";
+for my $i (0 .. 0xfe-0xa0) {
+ print "\t{";
+ for my $j (0 .. 0xfe-0xa0) {
+ if (defined $jis212[$i][$j]) {
+ printf "0x%04x", $jis212[$i][$j];
+ } else {
+ print "0x0000";
+ }
+ print ", " unless $j == 0xfe-0xa0;
+ }
+ print "}";
+ print ", " unless $i == 0xfe-0xa0;
+ print "\n";
+}
+print "};\n";
-- System Information:
Debian Release: testing/unstable
APT prefers unstable
APT policy: (990, 'unstable'), (950, 'unstable'), (900, 'experimental')
Architecture: i386 (i686)
Shell: /bin/sh linked to /bin/bash
Kernel: Linux 2.6.12
Locale: LANG=en_AU.UTF-8, LC_CTYPE=en_AU.UTF-8 (charmap=UTF-8)
Versions of packages lookup depends on:
ii libc6 2.3.5-6 GNU C Library: Shared libraries an
lookup recommends no packages.
-- no debconf information
--
Paul "TBBle" Hampson, [EMAIL PROTECTED]
8th year CompSci/Asian Studies student, ANU
Shorter .sig for a more eco-friendly paperless office.
pgpgYlpjCZKzX.pgp
Description: PGP signature
