CVS commit: [perseant-stdc-iso10646] src/lib/libc/citrus
Module Name:src Committed By: perseant Date: Mon Jan 29 22:08:59 UTC 2018 Modified Files: src/lib/libc/citrus [perseant-stdc-iso10646]: Makefile.inc citrus_trie.c citrus_trie.h src/lib/libc/citrus/modules [perseant-stdc-iso10646]: citrus_big5.c citrus_big5_k2u.h citrus_big5_u2k.h citrus_euc.c citrus_euc_k2u.h citrus_euc_u2k.h citrus_iso2022.c citrus_iso2022_k2u.h citrus_iso2022_u2k.h citrus_mskanji.c citrus_mskanji_k2u.h citrus_mskanji_u2k.h Log Message: Fix issues with trie implementation. Now passes the locale tests when __STDC_ISO_10646__ is defined, with the exception of tests requiring LC_COLLATE support. To generate a diff of this commit: cvs rdiff -u -r1.8.40.1 -r1.8.40.2 src/lib/libc/citrus/Makefile.inc cvs rdiff -u -r1.1.2.1 -r1.1.2.2 src/lib/libc/citrus/citrus_trie.c \ src/lib/libc/citrus/citrus_trie.h cvs rdiff -u -r1.15.18.4 -r1.15.18.5 \ src/lib/libc/citrus/modules/citrus_big5.c cvs rdiff -u -r1.1.2.1 -r1.1.2.2 \ src/lib/libc/citrus/modules/citrus_big5_k2u.h \ src/lib/libc/citrus/modules/citrus_big5_u2k.h \ src/lib/libc/citrus/modules/citrus_euc_k2u.h \ src/lib/libc/citrus/modules/citrus_euc_u2k.h \ src/lib/libc/citrus/modules/citrus_iso2022_k2u.h \ src/lib/libc/citrus/modules/citrus_iso2022_u2k.h \ src/lib/libc/citrus/modules/citrus_mskanji_k2u.h \ src/lib/libc/citrus/modules/citrus_mskanji_u2k.h cvs rdiff -u -r1.17.20.4 -r1.17.20.5 src/lib/libc/citrus/modules/citrus_euc.c cvs rdiff -u -r1.23.22.4 -r1.23.22.5 \ src/lib/libc/citrus/modules/citrus_iso2022.c cvs rdiff -u -r1.14.22.4 -r1.14.22.5 \ src/lib/libc/citrus/modules/citrus_mskanji.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. diffs are larger than 1MB and have been omitted
CVS commit: [perseant-stdc-iso10646] src/lib/libc/citrus
Module Name:src
Committed By: perseant
Date: Sun Jan 21 19:35:10 UTC 2018
Added Files:
src/lib/libc/citrus [perseant-stdc-iso10646]: citrus_trie.c
citrus_trie.h
Log Message:
Add files missing from previous commit
To generate a diff of this commit:
cvs rdiff -u -r0 -r1.1.2.1 src/lib/libc/citrus/citrus_trie.c \
src/lib/libc/citrus/citrus_trie.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Added files:
Index: src/lib/libc/citrus/citrus_trie.c
diff -u /dev/null src/lib/libc/citrus/citrus_trie.c:1.1.2.1
--- /dev/null Sun Jan 21 19:35:10 2018
+++ src/lib/libc/citrus/citrus_trie.c Sun Jan 21 19:35:10 2018
@@ -0,0 +1,404 @@
+#include "citrus_trie.h"
+#ifdef DEBUG_TRIE
+# include "modules/citrus_euc_data.h"
+# include "citrus_trie_test.h"
+#endif
+
+static void citrus_trie_dump_table_recursive(citrus_trie_node_t, size_t, FILE *, int);
+static void citrus_trie_load_table_recursive(citrus_trie_node_t, size_t, FILE *);
+static void citrus_trie_init_recursive(citrus_trie_node_t *, VALUE_TYPE *, int, int *, int *);
+
+citrus_trie_header_t
+citrus_trie_create(unsigned int flags, size_t bitwidth,
+ size_t nlevels, size_t off, size_t len)
+{
+ citrus_trie_header_t h;
+
+ h = (citrus_trie_header_t)malloc(sizeof(*h));
+ h->th_flags = flags;
+ h->th_bitwidth = bitwidth;
+ h->th_bitmask = (1 << bitwidth) - 1;
+ h->th_off = off;
+ h->th_level = nlevels - 1;
+#ifdef DEBUG_TRIE
+ h->th_size = sizeof(*h);
+#endif
+ h->th_root = citrus_trie_node_create(h, h->th_level, len);
+
+ return h;
+}
+
+citrus_trie_node_t
+citrus_trie_node_create(citrus_trie_header_t h, size_t level, size_t len)
+{
+ int i;
+ citrus_trie_node_t t;
+
+ t = (citrus_trie_node_t)citrus_trie_malloc(h, sizeof(*t));
+ t->tr_len = len;
+ if (len > 0) {
+ t->tr_u = (union citrus_trie_node_union *)citrus_trie_malloc(h, len * sizeof(*t->tr_u));
+ if (level == 0) {
+ for (i = 0; i < len; i++)
+t->tr_u[i].u_value = INVALID_VALUE;
+ } else {
+ for (i = 0; i < len; i++)
+t->tr_u[i].u_child = NULL;
+ }
+ } else
+ t->tr_u = NULL;
+
+ return t;
+}
+
+int
+citrus_trie_node_insert(citrus_trie_header_t h, citrus_trie_node_t t, size_t level, citrus_trie_key_t key, VALUE_TYPE value)
+{
+ size_t idx, i, olen;
+
+ idx = (key >> (level * h->th_bitwidth)) & h->th_bitmask;
+ if (idx < h->th_off)
+ return EINVAL;
+
+ idx -= h->th_off;
+ if (t->tr_len <= idx) {
+ olen = t->tr_len;
+ t->tr_len = idx + 1;
+#ifdef DEBUG_TRIE
+ h->th_size += (t->tr_len - olen) * sizeof(*t->tr_u);
+#endif
+ t->tr_u = (union citrus_trie_node_union *)realloc(t->tr_u, t->tr_len * sizeof(*t->tr_u));
+ for (i = olen; i < t->tr_len; i++) {
+ if (level == 0)
+t->tr_u[i].u_value = INVALID_VALUE;
+ else
+t->tr_u[i].u_child = NULL;
+ }
+ }
+
+ if (level == 0) {
+ t->tr_u[idx].u_value = value;
+ return 0;
+ } else {
+ if (t->tr_u[idx].u_child == NULL)
+ t->tr_u[idx].u_child = citrus_trie_node_create(h, level - 1, 0);
+ return citrus_trie_node_insert(h, t->tr_u[idx].u_child, level - 1,
+ key, value);
+ }
+}
+
+int
+citrus_trie_insert(citrus_trie_header_t h, citrus_trie_key_t key, VALUE_TYPE value)
+{
+ int r = citrus_trie_node_insert(h, h->th_root, h->th_level, key, value);
+#ifdef DEBUG_TRIE
+ int c = citrus_trie_lookup(h, key);
+ if (c != value)
+ fprintf(stderr, "Error on insert: key %x expected %x got %x\n", (unsigned)key, value, c);
+#endif
+ return r;
+}
+
+VALUE_TYPE
+citrus_trie_node_lookup(citrus_trie_header_t h, citrus_trie_node_t t, size_t level, citrus_trie_key_t key)
+{
+ size_t idx;
+
+ idx = (key >> (level * h->th_bitwidth)) & h->th_bitmask;
+ if (idx < h->th_off)
+ return INVALID_VALUE;
+
+ idx -= h->th_off;
+ if (idx >= t->tr_len)
+ return INVALID_VALUE;
+
+ if (level == 0) {
+ return t->tr_u[idx].u_value;
+ }
+ return citrus_trie_node_lookup(h, t->tr_u[idx].u_child, level - 1, key);
+}
+
+VALUE_TYPE
+citrus_trie_lookup(citrus_trie_header_t h, citrus_trie_key_t key)
+{
+ return citrus_trie_node_lookup(h, h->th_root, h->th_level, key);
+}
+
+/*
+ * Assume VALUE_TYPE flat[N][2].
+ */
+citrus_trie_header_t
+citrus_trie_create_from_flat(VALUE_TYPE *flat, size_t bitwidth, int count) {
+ VALUE_TYPE ne_key;
+ int i, j;
+ unsigned val;
+ citrus_trie_header_t h;
+ size_t bitmask = (1 << bitwidth) - 1;
+ size_t maxshift = (8 * sizeof(VALUE_TYPE)) / bitwidth;
+ size_t min = bitmask, max = 0, level = 0;
+
+ /* Loop through every element to see what
+ level, off and len should be */
+ for (i = 0; i < count; i++) {
+ ne_key = flat[i * 2];
+ for (j = 0; j < maxshift; j++) {
+ val = (ne_key >> (j * bitwidth)) & bitmask;
+ if (level < j + 1 && val)
+level = j + 1;
+ if (min > val)
+min = val;
+ if (max < val)
+max = val;
+ }
+ }
+
+ h = citrus_trie_create(0x0, bitwidth, level, min, max - min + 1);
+
+ /* Now add every value */
+ for (i = 0; i < count; i++) {
+ ne_key = flat[i * 2];
+
+ citrus_trie_insert(h, ne_key, flat[i * 2 +
CVS commit: [perseant-stdc-iso10646] src/lib/libc/citrus
Module Name:src
Committed By: perseant
Date: Mon Jul 31 04:23:35 UTC 2017
Modified Files:
src/lib/libc/citrus [perseant-stdc-iso10646]: citrus_ctype.h
citrus_ctype_template.h
src/lib/libc/citrus/modules [perseant-stdc-iso10646]: citrus_big5.c
citrus_euc.c citrus_iso2022.c citrus_mskanji.c
Log Message:
Make the compilation conditional on __STDC_ISO_10646__, so that applications
that need one of the East Asian locales but don't want __STDC_ISO_10646__
don't have to pay the penalty for it.
To generate a diff of this commit:
cvs rdiff -u -r1.3.22.1 -r1.3.22.2 src/lib/libc/citrus/citrus_ctype.h
cvs rdiff -u -r1.36.22.1 -r1.36.22.2 \
src/lib/libc/citrus/citrus_ctype_template.h
cvs rdiff -u -r1.15.18.2 -r1.15.18.3 \
src/lib/libc/citrus/modules/citrus_big5.c
cvs rdiff -u -r1.17.20.2 -r1.17.20.3 src/lib/libc/citrus/modules/citrus_euc.c
cvs rdiff -u -r1.23.22.2 -r1.23.22.3 \
src/lib/libc/citrus/modules/citrus_iso2022.c
cvs rdiff -u -r1.14.22.2 -r1.14.22.3 \
src/lib/libc/citrus/modules/citrus_mskanji.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/lib/libc/citrus/citrus_ctype.h
diff -u src/lib/libc/citrus/citrus_ctype.h:1.3.22.1 src/lib/libc/citrus/citrus_ctype.h:1.3.22.2
--- src/lib/libc/citrus/citrus_ctype.h:1.3.22.1 Fri Jul 21 20:22:29 2017
+++ src/lib/libc/citrus/citrus_ctype.h Mon Jul 31 04:23:35 2017
@@ -1,4 +1,4 @@
-/* $NetBSD: citrus_ctype.h,v 1.3.22.1 2017/07/21 20:22:29 perseant Exp $ */
+/* $NetBSD: citrus_ctype.h,v 1.3.22.2 2017/07/31 04:23:35 perseant Exp $ */
/*-
* Copyright (c)2002 Citrus Project,
@@ -30,6 +30,7 @@
#ifndef _CITRUS_CTYPE_H_
#define _CITRUS_CTYPE_H_
+#include /* For __STDC_ISO_10646__ */
#include "citrus_ctype_local.h"
typedef struct _citrus_ctype_rec *_citrus_ctype_t;
@@ -185,6 +186,7 @@ _citrus_ctype_wctob(_citrus_ctype_t cc,
return (*cc->cc_ops->co_wctob)(cc, c, cresult);
}
+#ifdef __STDC_ISO_10646__
static __inline int
_citrus_ctype_ucs2kt(_citrus_ctype_t cc,
wchar_kuten_t * __restrict ktp,
@@ -202,6 +204,11 @@ _citrus_ctype_kt2ucs(_citrus_ctype_t cc,
_DIAGASSERT(cc && cc->cc_ops && cc->cc_ops->co_kt2ucs);
return (*cc->cc_ops->co_kt2ucs)(cc->cc_closure, up, kt);
}
+#else
+/* Define away the calls to these functions */
+#define _citrus_ctype_ucs2kt(cl, ktp, wc) do {} while (0)
+#define _citrus_ctype_kt2ucs(cl, up, kt) do {} while (0)
+#endif
extern _citrus_ctype_rec_t _citrus_ctype_default;
Index: src/lib/libc/citrus/citrus_ctype_template.h
diff -u src/lib/libc/citrus/citrus_ctype_template.h:1.36.22.1 src/lib/libc/citrus/citrus_ctype_template.h:1.36.22.2
--- src/lib/libc/citrus/citrus_ctype_template.h:1.36.22.1 Fri Jul 21 20:22:29 2017
+++ src/lib/libc/citrus/citrus_ctype_template.h Mon Jul 31 04:23:35 2017
@@ -1,4 +1,4 @@
-/* $NetBSD: citrus_ctype_template.h,v 1.36.22.1 2017/07/21 20:22:29 perseant Exp $ */
+/* $NetBSD: citrus_ctype_template.h,v 1.36.22.2 2017/07/31 04:23:35 perseant Exp $ */
/*-
* Copyright (c)2002 Citrus Project,
@@ -118,6 +118,7 @@
*
*/
+#include /* For __STDC_ISO_10646__ */
/* prototypes */
@@ -938,6 +939,7 @@ _FUNCNAME(ctype_wctob)(_citrus_ctype_rec
return 0;
}
+#ifdef __STDC_ISO_10646__
static __inline int
/*ARGSUSED*/
_FUNCNAME(ctype_ucs2kt)(void * __restrict cl,
@@ -953,4 +955,8 @@ _FUNCNAME(ctype_kt2ucs)(void * __restric
{
return _FUNCNAME(kt2ucs)(_CEI_TO_EI(_TO_CEI(cl)), up, kt);
}
-
+#else
+/* Define away the calls to these functions */
+#define _FUNCNAME(ctype_ucs2kt)(cl, ktp, wc) do {} while (0)
+#define _FUNCNAME(ctype_kt2ucs)(cl, up, kt) do {} while (0)
+#endif
Index: src/lib/libc/citrus/modules/citrus_big5.c
diff -u src/lib/libc/citrus/modules/citrus_big5.c:1.15.18.2 src/lib/libc/citrus/modules/citrus_big5.c:1.15.18.3
--- src/lib/libc/citrus/modules/citrus_big5.c:1.15.18.2 Fri Jul 21 20:22:29 2017
+++ src/lib/libc/citrus/modules/citrus_big5.c Mon Jul 31 04:23:35 2017
@@ -1,4 +1,4 @@
-/* $NetBSD: citrus_big5.c,v 1.15.18.2 2017/07/21 20:22:29 perseant Exp $ */
+/* $NetBSD: citrus_big5.c,v 1.15.18.3 2017/07/31 04:23:35 perseant Exp $ */
/*-
* Copyright (c)2002, 2006 Citrus Project,
@@ -60,7 +60,7 @@
#include
#if defined(LIBC_SCCS) && !defined(lint)
-__RCSID("$NetBSD: citrus_big5.c,v 1.15.18.2 2017/07/21 20:22:29 perseant Exp $");
+__RCSID("$NetBSD: citrus_big5.c,v 1.15.18.3 2017/07/31 04:23:35 perseant Exp $");
#endif /* LIBC_SCCS and not lint */
#include
@@ -134,6 +134,7 @@ typedef struct {
#define _ENCODING_IS_STATE_DEPENDENT 0
#define _STATE_NEEDS_EXPLICIT_INIT(_ps_) 0
+#ifdef __STDC_ISO_10646__
#include "citrus_big5_data.h"
static __inline int
@@ -188,6 +189,9 @@ _FUNCNAME(kt2ucs)(_ENCODING_INFO * __res
return 0;
}
+#else
+#include "citrus_u2k_template.h"
+#endif
static __inline void
/*ARGSUSED*/
Index: src/lib/libc/citrus/modules/citrus_euc.c
diff -u src/lib/libc/citrus/modules/
