Hello again!
Patches
=======
I had implemented support for FLAG long, FLAG num and AF parameters. I
attached patch to the e-mail (hunspell-dict.patch).
This patch allow to use Hunspell dictionaries listed in the previous
e-mail: ar, br_fr, ca, ca_valencia, en_ca, en_gb, en_us, fr, gl_es,
hu_hu, is, ne_np, nl_nl, si_lk.
The most part of changes was in spell.c in the affix file parsing code.
The following are dictionary structures changes:
- useFlagAliases and flagMode fields had been added to the IspellDict
struct;
- flagval array size had been increased from 256 to 65000;
- flag field of the AFFIX struct also had been increased.
I also had implemented a patch that fixes an error from the e-mail
http://www.postgresql.org/message-id/562e1073.8030...@postgrespro.ru
This patch just ignore that error.
Tests
=====
Extention test dictionaries for loading into PostgreSQL and for
normalizing with ts_lexize function can be downloaded from
https://dl.dropboxusercontent.com/u/15423817/HunspellDictTest.tar.gz
It would be nice if somebody can do additional tests of dictionaries of
well known languages. Because I do not know many of them.
Other Improvements
==================
There are also some parameters for compound words. But I am not sure
that we want use this parameters.
--
Artur Zakirov
Postgres Professional: http://www.postgrespro.com
Russian Postgres Company
*** a/src/backend/tsearch/spell.c
--- b/src/backend/tsearch/spell.c
***************
*** 237,242 **** cmpaffix(const void *s1, const void *s2)
--- 237,309 ----
(const unsigned char *) a2->repl);
}
+ static unsigned short
+ decodeFlag(IspellDict *Conf, char *sflag, char **sflagnext)
+ {
+ unsigned short s;
+ char *next;
+
+ switch (Conf->flagMode)
+ {
+ case FM_LONG:
+ s = (int)sflag[0] << 8 | (int)sflag[1];
+ if (sflagnext)
+ *sflagnext = sflag + 2;
+ break;
+ case FM_NUM:
+ s = (unsigned short) strtol(sflag, &next, 10);
+ if (sflagnext)
+ {
+ if (next)
+ {
+ *sflagnext = next;
+ while (**sflagnext)
+ {
+ if (**sflagnext == ',')
+ {
+ *sflagnext = *sflagnext + 1;
+ break;
+ }
+ *sflagnext = *sflagnext + 1;
+ }
+ }
+ else
+ *sflagnext = 0;
+ }
+ break;
+ default:
+ s = (unsigned short) *((unsigned char *)sflag);
+ if (sflagnext)
+ *sflagnext = sflag + 1;
+ }
+
+ return s;
+ }
+
+ static bool
+ isAffixFlagInUse(IspellDict *Conf, int affix, unsigned short affixflag)
+ {
+ char *flagcur;
+ char *flagnext = 0;
+
+ if (affixflag == 0)
+ return true;
+
+ flagcur = Conf->AffixData[affix];
+
+ while (*flagcur)
+ {
+ if (decodeFlag(Conf, flagcur, &flagnext) == affixflag)
+ return true;
+ if (flagnext)
+ flagcur = flagnext;
+ else
+ break;
+ }
+
+ return false;
+ }
+
static void
NIAddSpell(IspellDict *Conf, const char *word, const char *flag)
{
***************
*** 355,361 **** FindWord(IspellDict *Conf, const char *word, int affixflag, int flag)
else if ((flag & StopMiddle->compoundflag) == 0)
return 0;
! if ((affixflag == 0) || (strchr(Conf->AffixData[StopMiddle->affix], affixflag) != NULL))
return 1;
}
node = StopMiddle->node;
--- 422,428 ----
else if ((flag & StopMiddle->compoundflag) == 0)
return 0;
! if (isAffixFlagInUse(Conf, StopMiddle->affix, affixflag))
return 1;
}
node = StopMiddle->node;
***************
*** 394,400 **** NIAddAffix(IspellDict *Conf, int flag, char flagflags, const char *mask, const c
Affix = Conf->Affix + Conf->naffixes;
! if (strcmp(mask, ".") == 0)
{
Affix->issimple = 1;
Affix->isregis = 0;
--- 461,467 ----
Affix = Conf->Affix + Conf->naffixes;
! if (strcmp(mask, ".") == 0 || *mask == '\0')
{
Affix->issimple = 1;
Affix->isregis = 0;
***************
*** 595,604 **** addFlagValue(IspellDict *Conf, char *s, uint32 val)
(errcode(ERRCODE_CONFIG_FILE_ERROR),
errmsg("multibyte flag character is not allowed")));
! Conf->flagval[*(unsigned char *) s] = (unsigned char) val;
Conf->usecompound = true;
}
/*
* Import an affix file that follows MySpell or Hunspell format
*/
--- 662,719 ----
(errcode(ERRCODE_CONFIG_FILE_ERROR),
errmsg("multibyte flag character is not allowed")));
! Conf->flagval[decodeFlag(Conf, s, (char **)NULL)] = (unsigned char) val;
Conf->usecompound = true;
}
+ static int
+ getFlagValues(IspellDict *Conf, char *s)
+ {
+ uint32 flag = 0;
+ char *flagcur;
+ char *flagnext = 0;
+
+ flagcur = s;
+ while (*flagcur)
+ {
+ flag |= Conf->flagval[decodeFlag(Conf, flagcur, &flagnext)];
+ if (flagnext)
+ flagcur = flagnext;
+ else
+ break;
+ }
+
+ return flag;
+ }
+
+ /*
+ * Get flag set from "s".
+ *
+ * Returns flag set from AffixData array if AF parameter used (useFlagAliases is true).
+ * In this case "s" is alias for flag set.
+ *
+ * Otherwise returns "s".
+ */
+ static char *
+ getFlags(IspellDict *Conf, char *s)
+ {
+ int curaffix;
+ if (Conf->useFlagAliases)
+ {
+ curaffix = strtol(s, (char **)NULL, 10);
+ if (curaffix && curaffix <= Conf->nAffixData)
+ /*
+ * Do not substract 1 from curaffix
+ * because empty string was added in NIImportOOAffixes
+ */
+ return Conf->AffixData[curaffix];
+ else
+ return VoidString;
+ }
+ else
+ return s;
+ }
+
/*
* Import an affix file that follows MySpell or Hunspell format
*/
***************
*** 615,621 **** NIImportOOAffixes(IspellDict *Conf, const char *filename)
char repl[BUFSIZ],
*prepl;
bool isSuffix = false;
! int flag = 0;
char flagflags = 0;
tsearch_readline_state trst;
int scanread = 0;
--- 730,740 ----
char repl[BUFSIZ],
*prepl;
bool isSuffix = false;
! int naffix = 0,
! curaffix = 0;
! int flag = 0,
! flagprev = 0,
! sflaglen = 0;
char flagflags = 0;
tsearch_readline_state trst;
int scanread = 0;
***************
*** 625,630 **** NIImportOOAffixes(IspellDict *Conf, const char *filename)
--- 744,751 ----
/* read file to find any flag */
memset(Conf->flagval, 0, sizeof(Conf->flagval));
Conf->usecompound = false;
+ Conf->useFlagAliases = false;
+ Conf->flagMode = FM_CHAR;
if (!tsearch_readline_begin(&trst, filename))
ereport(ERROR,
***************
*** 672,681 **** NIImportOOAffixes(IspellDict *Conf, const char *filename)
while (*s && t_isspace(s))
s += pg_mblen(s);
! if (*s && STRNCMP(s, "default") != 0)
! ereport(ERROR,
(errcode(ERRCODE_CONFIG_FILE_ERROR),
! errmsg("Ispell dictionary supports only default flag value")));
}
pfree(recoded);
--- 793,809 ----
while (*s && t_isspace(s))
s += pg_mblen(s);
! if (*s)
! {
! if (STRNCMP(s, "long") == 0)
! Conf->flagMode = FM_LONG;
! else if (STRNCMP(s, "num") == 0)
! Conf->flagMode = FM_NUM;
! else if (STRNCMP(s, "default") != 0)
! ereport(ERROR,
(errcode(ERRCODE_CONFIG_FILE_ERROR),
! errmsg("Ispell dictionary supports only default, long and num flag value")));
! }
}
pfree(recoded);
***************
*** 695,725 **** NIImportOOAffixes(IspellDict *Conf, const char *filename)
if (*recoded == '\0' || t_isspace(recoded) || t_iseq(recoded, '#'))
goto nextline;
scanread = sscanf(recoded, scanbuf, type, sflag, find, repl, mask);
if (ptype)
pfree(ptype);
ptype = lowerstr_ctx(Conf, type);
if (scanread < 4 || (STRNCMP(ptype, "sfx") && STRNCMP(ptype, "pfx")))
goto nextline;
! if (scanread == 4)
{
! if (strlen(sflag) != 1)
! goto nextline;
! flag = *sflag;
isSuffix = (STRNCMP(ptype, "sfx") == 0) ? true : false;
if (t_iseq(find, 'y') || t_iseq(find, 'Y'))
flagflags = FF_CROSSPRODUCT;
else
flagflags = 0;
}
else
{
char *ptr;
int aflg = 0;
! if (strlen(sflag) != 1 || flag != *sflag || flag == 0)
goto nextline;
prepl = lowerstr_ctx(Conf, repl);
/* affix flag */
--- 823,897 ----
if (*recoded == '\0' || t_isspace(recoded) || t_iseq(recoded, '#'))
goto nextline;
+ *find = *repl = *mask = '\0';
scanread = sscanf(recoded, scanbuf, type, sflag, find, repl, mask);
if (ptype)
pfree(ptype);
ptype = lowerstr_ctx(Conf, type);
+
+ /* First try to parse AF parameter (alias compression) */
+ if (STRNCMP(ptype, "af") == 0)
+ {
+ /* First line is the number of aliases */
+ if (!Conf->useFlagAliases)
+ {
+ Conf->useFlagAliases = true;
+ naffix = atoi(sflag);
+ if (naffix == 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ errmsg("invalid number of flag vector aliases")));
+
+ /* Also reserve place for empty flag set */
+ naffix++;
+
+ Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
+ Conf->lenAffixData = Conf->nAffixData = naffix;
+
+ /* Add empty flag set into AffixData */
+ Conf->AffixData[curaffix] = VoidString;
+ curaffix++;
+ }
+ /* Other lines is aliases */
+ else
+ {
+ if (curaffix < naffix)
+ {
+ Conf->AffixData[curaffix] = cpstrdup(Conf, sflag);
+ curaffix++;
+ }
+ }
+ goto nextline;
+ }
+ /* Else try to parse prefixes and suffixes */
if (scanread < 4 || (STRNCMP(ptype, "sfx") && STRNCMP(ptype, "pfx")))
goto nextline;
! sflaglen = strlen(sflag);
! if (sflaglen == 0
! || (sflaglen > 1 && Conf->flagMode == FM_CHAR)
! || (sflaglen > 2 && Conf->flagMode == FM_LONG))
! goto nextline;
! flag = decodeFlag(Conf, sflag, (char **)NULL);
!
! /* Affix header */
! if (flag != flagprev)
{
! flagprev = flag;
isSuffix = (STRNCMP(ptype, "sfx") == 0) ? true : false;
if (t_iseq(find, 'y') || t_iseq(find, 'Y'))
flagflags = FF_CROSSPRODUCT;
else
flagflags = 0;
}
+ /* Affix fields */
else
{
char *ptr;
int aflg = 0;
! if (flag == 0)
goto nextline;
prepl = lowerstr_ctx(Conf, repl);
/* affix flag */
***************
*** 727,737 **** NIImportOOAffixes(IspellDict *Conf, const char *filename)
{
*ptr = '\0';
ptr = repl + (ptr - prepl) + 1;
! while (*ptr)
! {
! aflg |= Conf->flagval[*(unsigned char *) ptr];
! ptr++;
! }
}
pfind = lowerstr_ctx(Conf, find);
pmask = lowerstr_ctx(Conf, mask);
--- 899,905 ----
{
*ptr = '\0';
ptr = repl + (ptr - prepl) + 1;
! aflg |= getFlagValues(Conf, getFlags(Conf, ptr));
}
pfind = lowerstr_ctx(Conf, find);
pmask = lowerstr_ctx(Conf, mask);
***************
*** 789,794 **** NIImportAffixes(IspellDict *Conf, const char *filename)
--- 957,964 ----
memset(Conf->flagval, 0, sizeof(Conf->flagval));
Conf->usecompound = false;
+ Conf->useFlagAliases = false;
+ Conf->flagMode = FM_CHAR;
while ((recoded = tsearch_readline(&trst)) != NULL)
{
***************
*** 931,946 **** MergeAffix(IspellDict *Conf, int a1, int a2)
static uint32
makeCompoundFlags(IspellDict *Conf, int affix)
{
! uint32 flag = 0;
! char *str = Conf->AffixData[affix];
!
! while (str && *str)
! {
! flag |= Conf->flagval[*(unsigned char *) str];
! str++;
! }
!
! return (flag & FF_DICTFLAGMASK);
}
static SPNode *
--- 1101,1108 ----
static uint32
makeCompoundFlags(IspellDict *Conf, int affix)
{
! char *str = Conf->AffixData[affix];
! return (getFlagValues(Conf, str) & FF_DICTFLAGMASK);
}
static SPNode *
***************
*** 1032,1071 **** NISortDictionary(IspellDict *Conf)
/* compress affixes */
! /* Count the number of different flags used in the dictionary */
!
! qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspellaffix);
!
! naffix = 0;
! for (i = 0; i < Conf->nspell; i++)
{
! if (i == 0 || strncmp(Conf->Spell[i]->p.flag, Conf->Spell[i - 1]->p.flag, MAXFLAGLEN))
! naffix++;
}
! /*
! * Fill in Conf->AffixData with the affixes that were used in the
! * dictionary. Replace textual flag-field of Conf->Spell entries with
! * indexes into Conf->AffixData array.
! */
! Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
! curaffix = -1;
! for (i = 0; i < Conf->nspell; i++)
! {
! if (i == 0 || strncmp(Conf->Spell[i]->p.flag, Conf->AffixData[curaffix], MAXFLAGLEN))
{
! curaffix++;
! Assert(curaffix < naffix);
! Conf->AffixData[curaffix] = cpstrdup(Conf, Conf->Spell[i]->p.flag);
}
! Conf->Spell[i]->p.d.affix = curaffix;
! Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word);
}
- Conf->lenAffixData = Conf->nAffixData = naffix;
-
qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspell);
Conf->Dictionary = mkSPNode(Conf, 0, Conf->nspell, 0);
}
--- 1194,1249 ----
/* compress affixes */
! /* If we use flag aliases then we need to use Conf->AffixData filled in NIImportOOAffixes */
! if (Conf->useFlagAliases)
{
! for (i = 0; i < Conf->nspell; i++)
! {
! curaffix = strtol(Conf->Spell[i]->p.flag, (char **)NULL, 10);
! if (curaffix && curaffix <= Conf->nAffixData)
! Conf->Spell[i]->p.d.affix = curaffix;
! else
! Conf->Spell[i]->p.d.affix = 0;
! Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word);
! }
}
+ /* Otherwise fill Conf->AffixData here */
+ else
+ {
+ /* Count the number of different flags used in the dictionary */
+ qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspellaffix);
! naffix = 0;
! for (i = 0; i < Conf->nspell; i++)
! {
! if (i == 0 || strncmp(Conf->Spell[i]->p.flag, Conf->Spell[i - 1]->p.flag, MAXFLAGLEN))
! naffix++;
! }
! /*
! * Fill in Conf->AffixData with the affixes that were used in the
! * dictionary. Replace textual flag-field of Conf->Spell entries with
! * indexes into Conf->AffixData array.
! */
! Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
!
! curaffix = -1;
! for (i = 0; i < Conf->nspell; i++)
{
! if (i == 0 || strncmp(Conf->Spell[i]->p.flag, Conf->AffixData[curaffix], MAXFLAGLEN))
! {
! curaffix++;
! Assert(curaffix < naffix);
! Conf->AffixData[curaffix] = cpstrdup(Conf, Conf->Spell[i]->p.flag);
! }
!
! Conf->Spell[i]->p.d.affix = curaffix;
! Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word);
}
! Conf->lenAffixData = Conf->nAffixData = naffix;
}
qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspell);
Conf->Dictionary = mkSPNode(Conf, 0, Conf->nspell, 0);
}
***************
*** 1185,1196 **** mkVoidAffix(IspellDict *Conf, bool issuffix, int startsuffix)
}
static bool
! isAffixInUse(IspellDict *Conf, char flag)
{
int i;
for (i = 0; i < Conf->nAffixData; i++)
! if (strchr(Conf->AffixData[i], flag) != NULL)
return true;
return false;
--- 1363,1374 ----
}
static bool
! isAffixInUse(IspellDict *Conf, int flag)
{
int i;
for (i = 0; i < Conf->nAffixData; i++)
! if (isAffixFlagInUse(Conf, i, flag))
return true;
return false;
***************
*** 1219,1225 **** NISortAffixes(IspellDict *Conf)
firstsuffix = i;
if ((Affix->flagflags & FF_COMPOUNDFLAG) && Affix->replen > 0 &&
! isAffixInUse(Conf, (char) Affix->flag))
{
if (ptr == Conf->CompoundAffix ||
ptr->issuffix != (ptr - 1)->issuffix ||
--- 1397,1403 ----
firstsuffix = i;
if ((Affix->flagflags & FF_COMPOUNDFLAG) && Affix->replen > 0 &&
! isAffixInUse(Conf, Affix->flag))
{
if (ptr == Conf->CompoundAffix ||
ptr->issuffix != (ptr - 1)->issuffix ||
***************
*** 1685,1691 **** SplitToVariants(IspellDict *Conf, SPNode *snode, SplitVar *orig, char *word, int
if (StopLow < StopHigh)
{
! if (level == FF_COMPOUNDBEGIN)
compoundflag = FF_COMPOUNDBEGIN;
else if (level == wordlen - 1)
compoundflag = FF_COMPOUNDLAST;
--- 1863,1869 ----
if (StopLow < StopHigh)
{
! if (startpos == 0)
compoundflag = FF_COMPOUNDBEGIN;
else if (level == wordlen - 1)
compoundflag = FF_COMPOUNDLAST;
*** a/src/include/tsearch/dicts/spell.h
--- b/src/include/tsearch/dicts/spell.h
***************
*** 77,83 **** typedef struct spell_struct
typedef struct aff_struct
{
! uint32 flag:8,
type:1,
flagflags:7,
issimple:1,
--- 77,83 ----
typedef struct aff_struct
{
! uint32 flag:16,
type:1,
flagflags:7,
issimple:1,
***************
*** 132,137 **** typedef struct
--- 132,144 ----
bool issuffix;
} CMPDAffix;
+ typedef enum
+ {
+ FM_CHAR,
+ FM_LONG,
+ FM_NUM
+ } FlagMode;
+
typedef struct
{
int maffixes;
***************
*** 145,155 **** typedef struct
char **AffixData;
int lenAffixData;
int nAffixData;
CMPDAffix *CompoundAffix;
! unsigned char flagval[256];
bool usecompound;
/*
* Remaining fields are only used during dictionary construction; they are
--- 152,164 ----
char **AffixData;
int lenAffixData;
int nAffixData;
+ bool useFlagAliases;
CMPDAffix *CompoundAffix;
! unsigned char flagval[65000];
bool usecompound;
+ FlagMode flagMode;
/*
* Remaining fields are only used during dictionary construction; they are
*** a/src/backend/tsearch/spell.c
--- b/src/backend/tsearch/spell.c
***************
*** 429,443 **** NIAddAffix(IspellDict *Conf, int flag, char flagflags, const char *mask, const c
err = pg_regcomp(&(Affix->reg.regex), wmask, wmasklen,
REG_ADVANCED | REG_NOSUB,
DEFAULT_COLLATION_OID);
if (err)
! {
! char errstr[100];
!
! pg_regerror(err, &(Affix->reg.regex), errstr, sizeof(errstr));
! ereport(ERROR,
! (errcode(ERRCODE_INVALID_REGULAR_EXPRESSION),
! errmsg("invalid regular expression: %s", errstr)));
! }
}
Affix->flagflags = flagflags;
--- 429,437 ----
err = pg_regcomp(&(Affix->reg.regex), wmask, wmasklen,
REG_ADVANCED | REG_NOSUB,
DEFAULT_COLLATION_OID);
+ /* Ignore regular expression error and do not add wrong affix */
if (err)
! return;
}
Affix->flagflags = flagflags;
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
