able/windows-nt
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
windows-nt/Source/XPSP1/NT/enduser/speech/common/spcommon/ltscart.cpp

1236 lines
29 KiB
C++
Raw Normal View History

Add source files
2020-09-26 03:20:57 -05:00
/*******************************************************************************
* LtsCart.cpp *
*----------*
*
* ** WARNING **
* CART code for LTS. This code was created in MS Research and LiJ owns
* the algorithm. YunusM eliminated the private heap used by this code
* and used the new and delete operators instead.
*
* Created By: LIJ (MS Research) Date: 06/18/99
* Current Owner: Fil
*
* Copyright (C) 1999 Microsoft Corporation. All Rights Reserved
*******************************************************************************/
//--- Includes --------------------------------------------------------------
#include "StdAfx.h"
#include "LtsCart.h"
#pragma warning(disable : 4100)
/* the following are for exceptions: single letter and NULL output */
static const char *bogus_pron_1033 = "B OW G AH S P R AH N AH N S IY EY SH AH N";
static const char *bogus_pron_1041 = "N A N I"; // what?
static const char *single_letter_pron_1033[52] =
{
"EY",
"B IY",
"S IY",
"D IY",
"IY",
"EH F",
"JH IY",
"EY CH",
"AY",
"JH EY",
"K EY",
"EH L",
"EH M",
"EH N",
"OW",
"P IY",
"K Y UW",
"AA R",
"EH S",
"T IY",
"Y UW",
"V IY",
"D AH B AX L Y UW",
"EH K S",
"W AY",
"Z IY",
//
// PLURAL SPELLINGS
//
"EY Z",
"B IY Z",
"S IY Z",
"D IY Z",
"IY Z",
"EH F S",
"JH IY Z",
"EY CH AX Z",
"AY Z",
"JH EY Z",
"K EY Z",
"EH L Z",
"EH M Z",
"EH N Z",
"OW Z",
"P IY Z",
"K Y UW Z",
"AA R Z",
"EH S AX Z",
"T IY Z",
"Y UW Z",
"V IY Z",
"D AH B AX L Y UW Z",
"EH K S AX Z",
"W AY Z",
"Z IY Z"
};
static const char *single_letter_pron_1041[52] =
{
"EE",
"B II",
"SH II",
"D II",
"II",
"E H U",
"J II",
"EE CH I",
"A I",
"J EE",
"K EE",
"E R U",
"E M U",
"E N U",
"OO",
"P II",
"K Y UU",
"AA R U",
"E S U",
"T II",
"Y UU",
"B U I",
"D A B U R Y UU",
"E STOP K U S U",
"W A I",
"Z E STOP T O",
//
// PLURAL SPELLINGS
//
"EE Z U",
"B II Z U",
"SH II Z U",
"D II Z U",
"II Z U",
"E H U Z U",
"J II Z U",
"EE CH I Z U",
"A I Z U",
"J EE Z U",
"K EE Z U",
"E R U Z U",
"E M U Z U",
"E N U Z U",
"OO Z U",
"P II Z U",
"K Y UU Z U",
"AA R U Z U",
"E S U Z U",
"T II Z U",
"Y UU Z U",
"B U I Z U",
"D A B U R Y UU Z U",
"E STOP K U S U Z U",
"W A I Z U",
"Z E STOP T O Z U"
};
/*
* not worthwhile to use binary search with only about 30 entries
*/
static int symbol_to_id(LTS_SYMTAB *tab, char *sym)
{
USES_CONVERSION;
SPDBG_FUNC("symbol_to_id");
int i;
for (i = 0; i < tab->n_symbols; i++)
{
if (CSTR_EQUAL == CompareString(MAKELCID(MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), SORT_DEFAULT), NORM_IGNORECASE,
A2T(tab->storage + tab->sym_idx[i]), -1, A2T(sym), -1))
{
return i;
}
}
return NO_SYMBOL;
} // static int symbol_to_id(LTS_SYMTAB *tab, char *sym)
static char *id_to_symbol(LTS_SYMTAB *tab, int id)
{
SPDBG_FUNC("id_to_symbol");
if (id < 0 || id > tab->n_symbols)
{
return NULL;
}
else
{
return tab->storage + tab->sym_idx[id];
}
} // static char *id_to_symbol(LTS_SYMTAB *tab, int id)
__inline void ODS (const char *format, ...)
{
#ifdef _DEBUG
SPDBG_FUNC("ODS");
va_list arglist;
va_start (arglist, format);
char buf[2048];
_vsnprintf(buf, 2048, format, arglist);
OutputDebugStringA(buf);
va_end (arglist);
#endif
}
__inline int ans_simp_question (LTS_FEATURE *feat, SIMPLE_QUESTION question,
LTS_SAMPLE *sample)
{
SPDBG_FUNC("ans_simp_question");
SYMBOL id;
int *phones = feat[question.questype].feature[question.feature];
SAMPLE_GET_CONTEXT(sample, question.questype, question.context,
question.offset, id);
return (TST_BIT(phones, id) ? TRUE : FALSE);
} // __inline int ans_simp_question (LTS_FEATURE *feat, SIMPLE_QUESTION question,
static int product_eval (LTS_FEATURE *feat, char *term, LTS_SAMPLE *sample)
{
SPDBG_FUNC("product_eval");
int negate, result;
SIMPLE_QUESTION ques;
char *cptr;
cptr = term;
while (TRUE)
{
/* negation sign */
if (*cptr == '~')
{
negate = TRUE;
cptr++;
}
else
{
negate = FALSE;
}
if (!isdigit(*cptr))
{
//quit (-1, "Invalid product in product_eval\n");
// OutputDebugString("Invalid product in product_eval\n");
return FALSE;
}
for (result = *cptr++ - '0'; isdigit (*cptr); cptr++)
{
result = result * 10 + (*cptr - '0');
}
QUES_DECODE(result, ques.questype, ques.context, ques.offset,
ques.feature);
if ((negate ^ ans_simp_question (feat, ques, sample)) == FALSE)
{
return FALSE;
}
if (*cptr == '\0')
{
break;
}
if (*cptr++ != '&')
{
//quit (-1, "product_eval: syntax error in product term %s\n", term);
/*
char szTemp[512];
sprintf(szTemp, "product_eval: syntax error in product term %s\n", term);
OutputDebugString(szTemp);
*/
return FALSE;
}
}
return TRUE;
} // static int product_eval (LTS_FEATURE *feat, char *term, LTS_SAMPLE *sample)
static int ans_comp_question(LTS_FEATURE *feat, char *prod,
LTS_SAMPLE *sample)
{
SPDBG_FUNC("ans_comp_question");
int i, num_products, limit;
char *cptr, string[LONGEST_STR], *products[MAX_PRODUCTS];
strcpy(string, prod);
for (cptr = string, num_products = 1; *cptr != '\0'; cptr++)
{
if (*cptr == '|') num_products++;
}
if (num_products > MAX_PRODUCTS)
{
//quit(1, "please increase MAX_PRODUCTS up to %d at least\n", num_products);
/*
char szTemp[256];
sprintf(szTemp, "please increase MAX_PRODUCTS up to %d at least\n", num_products);
OutputDebugString(szTemp);
*/
return FALSE;
}
for (i = 0, limit = num_products -1, cptr = string; ; i++)
{
products[i] = cptr++;
if (i == limit)
{
break;
}
for (; *cptr != '|'; cptr++) {};
*cptr++ = '\0';
}
for (i = 0; i < num_products; i++)
{
if (product_eval (feat, products[i], sample) == TRUE)
{
return TRUE;
}
}
return FALSE;
} // static int ans_comp_question(LTS_FEATURE *feat, char *prod,
static T_NODE *find_leaf(LTS_FEATURE *feat, T_NODE *root, LTS_SAMPLE *sample)
{
SPDBG_FUNC("find_leaf");
if (!root->yes_child)
{
return root;
}
else if (ans_comp_question(feat, root->prod, sample))
{
return find_leaf(feat, root->yes_child, sample);
}
else
{
return find_leaf(feat, root->no_child, sample);
}
} // static T_NODE *find_leaf(LTS_FEATURE *feat, T_NODE *root, LTS_SAMPLE *sample)
static int lts_product_eval (LTS_FEATURE *feat, LTS_PROD *term,
LTS_SAMPLE *sample, LTS_PROD **next)
{
SPDBG_FUNC("lts_product_eval");
int negate, result;
SIMPLE_QUESTION ques;
LTS_PROD *cptr = term;
while (TRUE)
{
if ((*cptr) & PROD_NEG)
{
negate = TRUE;
result = (*cptr) ^ PROD_NEG;
}
else
{
negate = FALSE;
result = (*cptr);
}
QUES_DECODE(result, ques.questype, ques.context, ques.offset,
ques.feature);
if ((negate ^ ans_simp_question (feat, ques, sample)) == FALSE)
{
while (*cptr != PROD_TERM && *cptr != QUES_TERM)
{
cptr++;
}
if (*cptr == QUES_TERM)
{
*next = NULL;
}
else
{
*next = cptr + 1;
}
return FALSE;
}
cptr++;
if (*cptr == QUES_TERM)
{
*next = NULL;
break;
}
else if (*cptr == PROD_TERM)
{
*next = cptr + 1;
break;
}
}
return TRUE;
} // static int lts_product_eval (LTS_FEATURE *feat, LTS_PROD *term,
static int lts_ans_comp_question(LTS_TREE UNALIGNED *tree, LTS_FEATURE *feat,
int idx, LTS_SAMPLE *sample)
{
SPDBG_FUNC("lts_ans_comp_question");
LTS_PROD *next, *term = (LTS_PROD *) ((char *) tree->p_prod + idx);
while (TRUE)
{
if (lts_product_eval (feat, term, sample, &next) == TRUE)
{
return TRUE;
}
if (next == NULL)
{
break;
}
term = next;
}
return FALSE;
} // static int lts_ans_comp_question(LTS_TREE *tree, LTS_FEATURE *feat,
static LTS_NODE *lts_find_leaf(LTS_TREE UNALIGNED *tree, LTS_FEATURE *feat,
LTS_NODE *root, LTS_SAMPLE *sample)
{
SPDBG_FUNC("lts_find_leaf");
if (IS_LEAF_NODE(root))
{
return root;
}
else if (lts_ans_comp_question(tree, feat, ((LTS_NODE UNALIGNED *)root)->idx, sample))
{
return lts_find_leaf(tree, feat, root + ((LTS_NODE UNALIGNED *)root)->yes, sample);
}
else
{
return lts_find_leaf(tree, feat, root + ((LTS_NODE UNALIGNED *)root)->yes + 1, sample);
}
} // static LTS_NODE *lts_find_leaf(LTS_TREE *tree, LTS_FEATURE *feat,
static LTS_DIST *lts_find_leaf_count(LTS_FOREST *l_forest, SYMBOL *pIn,
SYMBOL *pOut)
{
SPDBG_FUNC("lts_find_leaf_count");
LTS_TREE UNALIGNED *tree = l_forest->tree[*pIn];
LTS_NODE UNALIGNED *leaf;
LTS_SAMPLE sample;
/*
* construct a sample in order to share all the code with training
*/
sample.pIn = pIn;
sample.pOut = pOut;
/* *pOut cannot be NULL_SYMBOL_ID */
*pOut = NULL_SYMBOL_ID + 1;
leaf = lts_find_leaf(tree, l_forest->features, &(tree->nodes[0]), &sample);
return (LTS_DIST *) ((char *)tree->p_dist + leaf->idx);
} // static LTS_DIST *lts_find_leaf_count(LTS_FOREST *l_forest, SYMBOL *pIn,
static LTS_OUT_RESULT *allocate_out_result(LTS_FOREST *l_forest)
{
SPDBG_FUNC("allocate_out_result");
LTS_OUT_RESULT *res = new LTS_OUT_RESULT;
if (res)
{
res->out_strings = new LTS_OUT_STRING *[MAX_ALT_STRINGS];
if (res->out_strings)
{
res->num_allocated_strings = MAX_ALT_STRINGS;
res->num_strings = 0;
}
else
{
delete res;
res = NULL;
}
}
return res;
} // static LTS_OUT_RESULT *allocate_out_result(LTS_FOREST *l_forest)
static void free_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res)
{
SPDBG_FUNC("free_out_result");
int i;
for (i = 0; i < res->num_strings; i++)
{
delete res->out_strings[i];
}
if (res->num_allocated_strings == MAX_ALT_STRINGS)
{
delete res->out_strings;
}
else
{
free(res->out_strings); /* dirty */
}
delete res;
} // static void free_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res)
static bool reallocate_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res,
int min)
{
SPDBG_FUNC("reallocate_out_result");
int s = res->num_allocated_strings, old_size = s;
LTS_OUT_STRING **p;
while (s < min)
s += INC_ALT_STRINGS;
p = res->out_strings;
res->out_strings = (LTS_OUT_STRING **)
calloc(s, sizeof(LTS_OUT_STRING *));
if (!res->out_strings)
{
return false;
}
memcpy(res->out_strings, p, old_size * sizeof(LTS_OUT_STRING *));
if (old_size == MAX_ALT_STRINGS)
{
delete p;
}
else
{
free(p);
}
res->num_allocated_strings = s;
ODS("increased out_strings to %d in order to meet %d\n", s, min);
return true;
} // static void reallocate_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res,
static bool grow_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res,
SYMBOL i, int count, float inv_sum,
LTS_OUT_RESULT *tmpRes)
{
SPDBG_FUNC("grow_out_result");
int j;
if (res->num_strings + tmpRes->num_strings >= res->num_allocated_strings)
{
if (!reallocate_out_result(l_forest, res,
res->num_strings + tmpRes->num_strings))
{
return false;
}
}
for (j = 0; j < tmpRes->num_strings; j++)
{
SYMBOL *psrc = tmpRes->out_strings[j]->psym;
SYMBOL *ptgt;
res->out_strings[res->num_strings + j] = new LTS_OUT_STRING;
if (!res->out_strings)
{
return false;
}
ptgt = res->out_strings[res->num_strings + j]->psym;
*ptgt++ = i;
while (*psrc != NULL_SYMBOL_ID)
{
*ptgt++ = *psrc++;
}
*ptgt++ = NULL_SYMBOL_ID;
res->out_strings[res->num_strings + j]->prob = count * inv_sum *
tmpRes->out_strings[j]->prob;
}
res->num_strings += tmpRes->num_strings;
free_out_result(l_forest, tmpRes);
return true;
} // static void grow_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res,
static LTS_OUT_RESULT *gen_one_output(LTS_FOREST *l_forest, int len,
SYMBOL *input_id, int in_index,
SYMBOL *output_id, float cutoff)
{
SPDBG_FUNC("gen_one_output");
SYMBOL out[SP_MAX_WORD_LENGTH], *pOut;
LTS_OUT_RESULT *res = allocate_out_result(l_forest);
if (!res)
{
return NULL;
}
int sum, i, dim;
LTS_DIST UNALIGNED *pdf;
LTS_PAIR UNALIGNED *l_pair, *lp;
float cut, inv_sum;
/*
* copy output_id to local
*/
SYMBOL *psrc = output_id - 1, *ptgt = out;
while (*psrc != NULL_SYMBOL_ID) psrc--;
while (psrc != output_id)
*ptgt++ = *psrc++;
pOut = ptgt;
/* sanity check */
if (pOut - out != in_index + 1)
{
// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
int *z=0;
z[0]=z[1];
}
if (in_index == len - 1)
{
pdf = lts_find_leaf_count(l_forest, input_id + in_index, pOut);
l_pair = &(pdf->p_pair);
dim = pdf->c_dists;
for (lp = l_pair, sum = 0, i = 0; i < dim; i++, lp++)
{
sum += lp->cnt;
}
SPDBG_ASSERT(sum > 0);
inv_sum = 1.0f / sum;
cut = cutoff * sum;
for (lp = l_pair, i = 0; i < dim; i++, lp++)
{
if ((float)(lp->cnt) > cut)
{
res->out_strings[res->num_strings] = new LTS_OUT_STRING;
if (NULL == res->out_strings[res->num_strings])
{
return NULL;
}
res->out_strings[res->num_strings]->psym[0] = (SYMBOL) lp->id;
res->out_strings[res->num_strings]->psym[1] = NULL_SYMBOL_ID;
res->out_strings[res->num_strings]->prob = lp->cnt * inv_sum;
res->num_strings++;
} /* cut */
}
}
else
{
LTS_OUT_RESULT *tmpRes;
pdf = lts_find_leaf_count(l_forest, input_id + in_index, pOut);
dim = pdf->c_dists;
l_pair = &(pdf->p_pair);
for (lp = l_pair, sum = 0, i = 0; i < dim; i++, lp++)
{
sum += lp->cnt;
}
SPDBG_ASSERT(sum > 0);
inv_sum = 1.0f / sum;
cut = cutoff * sum;
for (lp = l_pair, i = 0; i < dim; i++, lp++)
{
if ((float)(lp->cnt) > cut)
{
SYMBOL *pTmpOut = pOut + 1;
*pOut = (SYMBOL) lp->id;
tmpRes = gen_one_output(l_forest, len, input_id, in_index + 1, pTmpOut, cutoff);
if (!tmpRes)
{
return NULL;
}
if (!grow_out_result(l_forest, res, (SYMBOL)(lp->id), lp->cnt,
inv_sum, tmpRes))
{
return NULL;
}
}
} /* i */
} /* else */
return res;
} // static LTS_OUT_RESULT *gen_one_output(LTS_FOREST *l_forest, int len,
static int comp_out_result_prob(const void *vp1, const void *vp2)
{
SPDBG_FUNC("comp_out_result_prob");
LTS_OUT_STRING **p1 = (LTS_OUT_STRING **) vp1,
**p2 = (LTS_OUT_STRING **) vp2;
if ((*p1)->prob > (*p2)->prob)
{
return -1;
}
else if ((*p1)->prob < (*p2)->prob)
{
return 1;
}
else
{
return 0;
}
} // static int comp_out_result_prob(const void *vp1, const void *vp2)
static void lts_fill_out_buffer(LTS_FOREST *l_forest, LTS_OUT_RESULT *out,
char *word)
{
SPDBG_FUNC("lts_fill_out_buffer");
int i, j, n;
float inv_sum, sum = 0.0f;
char phnstr[LONGEST_STR];
char *tmp;
LTS_SYMTAB *tab = l_forest->symbols;
if (out == NULL)
{
return;
}
if (word)
{
strcpy(l_forest->out.word, word);
}
else
{
l_forest->out.word[0] = 0;
}
/* normalize probabilities */
for (i = 0; i < out->num_strings; i++)
{
sum += out->out_strings[i]->prob;
}
inv_sum = 1.0f / sum;
for (i = 0; i < out->num_strings; i++)
{
out->out_strings[i]->prob *= inv_sum;
}
/*
* sort them according to the prob field
*/
qsort(out->out_strings, out->num_strings, sizeof(LTS_OUT_STRING *),
&comp_out_result_prob);
if (out->num_strings > MAX_OUTPUT_STRINGS - l_forest->out.num_prons)
{
n = MAX_OUTPUT_STRINGS - l_forest->out.num_prons;
for (sum = 0.0f, i = 0; i < n; i++)
{
sum += out->out_strings[i]->prob;
}
inv_sum = 1.0f / sum;
for (i = 0; i < n; i++)
{
out->out_strings[i]->prob *= inv_sum;
}
}
else
{
n = out->num_strings;
}
for (j = l_forest->out.num_prons, i = 0; i < n; i++)
{
SYMBOL *p = out->out_strings[i]->psym;
char *psrc, *ptgt;
if (out->out_strings[i]->prob < MIN_OUT_PROB)
{
continue;
}
phnstr[0] = 0;
l_forest->out.pron[j].prob = out->out_strings[i]->prob;
while (*p != NULL_SYMBOL_ID)
{
tmp = id_to_symbol(&(tab[OUTPUT]), *p++);
SPDBG_ASSERT(tmp);
if (tmp)
{
strcat(phnstr, tmp);
strcat(phnstr, " ");
}
}
psrc = phnstr;
ptgt = l_forest->out.pron[j].pstr;
while (*psrc)
{
if (*psrc != '#' && *psrc != '_')
{
*ptgt++ = *psrc++;
}
else if (*psrc == '_')
{
*ptgt++ = ' ';
psrc++;
}
else
{
psrc += 2; /* skip an extra space */
}
/* extreme case, truncate it */
if (ptgt - l_forest->out.pron[j].pstr >= SP_MAX_PRON_LENGTH)
{
for (ptgt--; !isspace(*ptgt); ptgt--) {}; /* never output partial phone */
ptgt++;
break;
}
}
// output could contain only '# '
if (ptgt > l_forest->out.pron[j].pstr && *(ptgt - 1) == ' ')
{
*(ptgt - 1) = 0; /* remove the last space */
}
else
{
*ptgt = 0; /* shouldn't happen unless ptgt didn't move */
}
if (ptgt > l_forest->out.pron[j].pstr)
{
j++;
}
} /* i */
if (j <= MAX_OUTPUT_STRINGS)
{
l_forest->out.num_prons = j;
}
else
{
l_forest->out.num_prons = MAX_OUTPUT_STRINGS; // should never happen
}
free_out_result(l_forest, out);
} // static void lts_fill_out_buffer(LTS_FOREST *l_forest, LTS_OUT_RESULT *out,
void assign_a_fixed_pron(LTS_OUTPUT *out, const char *pron, char *word)
{
SPDBG_FUNC("assign_a_fixed_pron");
out->num_prons = 1;
strcpy(out->word, word);
out->pron[0].prob = 1.0f;
if (strlen(pron) < SP_MAX_PRON_LENGTH)
{
strcpy(out->pron[0].pstr, pron);
}
else
{
char *p;
strncpy(out->pron[0].pstr, pron, SP_MAX_PRON_LENGTH);
p = &(out->pron[0].pstr[SP_MAX_PRON_LENGTH - 1]);
while (!isspace(*p))
{
p--; /* truncate the last partial phoneme */
}
*p = 0;
}
} // void assign_a_fixed_pron(LTS_OUTPUT *out, char *pron, char *word)
inline BOOL IsCharInRangeA(int ch, int chMin, int chMax)
{
return (unsigned)(ch - chMin) <= (unsigned)(chMax - chMin);
}
void assign_a_spelling_pron(LTS_OUTPUT *out, const char * single_letter_pron[52], char *word)
{
SPDBG_FUNC("assign_a_spelling_pron");
char *p;
int cchPron = 0;
strcpy(out->word, word);
if (ispunct(*word))
{
p = word + 1;
}
else
{
p = word;
}
out->num_prons = 1;
out->pron[0].prob = 1.0f;
out->pron[0].pstr[0] = 0;
char * pchPron = out->pron[0].pstr;
while (*p)
{
int cPOffset = 0; // 0 for single letter, 26 for plurals
int c = *p++;
// Lowercaseify, and skip over non-letters
if (IsCharInRangeA(c, 'A', 'Z'))
{
c += 'a' - 'A';
}
else if (!IsCharInRangeA(c, 'a', 'z'))
{
continue;
}
// Check if the next two characters are 'S (apostrophe S). Include the following cases: words ending in 's 'S s' S'
// If they are we use a the plural pronunciation for the letter and skip over the letter and 'S
if ((p[0] == '\'') && ((0 == p[1] && 's' == c) || 's' == p[1] || 'S' == p[1]))
{
cPOffset = 26;
p += p[1] ? 1 : 0; // skip 'S
}
// Make sure the string isn't too long accounting for the new phone and seperator
const char * const pchPronT = single_letter_pron[cPOffset + c - 'a'];
const int cchPronT = strlen(pchPronT);
if ((cchPron + 1 + cchPronT) < (SP_MAX_PRON_LENGTH - 1)) // +1 for separating space, -1 for terminating NUL
{
strcpy(pchPron + cchPron, pchPronT);
cchPron += cchPronT;
pchPron[cchPron++] = ' ';
}
else
{
break;
}
}
if (cchPron)
{
pchPron[cchPron - 1] = 0; // trim trailing space char
}
}
HRESULT LtscartGetPron(LTS_FOREST *l_forest, char *word, LTS_OUTPUT **ppLtsOutput)
{
SPDBG_FUNC("LtscartGetPron");
HRESULT hr = S_OK;
LTS_OUT_RESULT *pres = NULL;
char *p, *base;
SYMBOL buffer[LONGEST_STR], *pbuf = buffer + 1;
int len, id, hasvowel = 0, allcapital = 1;
l_forest->out.num_prons = 0;
buffer[0] = NULL_SYMBOL_ID;
len = 0;
if (word == NULL || (base = strtok(word, " \t\n")) == NULL)
{
assign_a_fixed_pron(&(l_forest->out), l_forest->bogus_pron, "NUL");
*ppLtsOutput = &(l_forest->out);
return S_FALSE;
}
else
{
base = strtok(word, " \t\n");
if (ispunct(*base))
{
for (p = base; *p && ispunct(*p); p++) {};
}
else
{
p = base;
}
}
char ach[2];
ach[1] = 0;
while (*p)
{
const int d = *p++;
const int c = tolower(d);
if (!hasvowel && (c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u' || c == 'y'))
{
hasvowel = 1;
}
if (allcapital && d == c)
{
allcapital = 0;
}
ach[0] = (char)c;
if ((id = symbol_to_id (&(l_forest->symbols[INPUT]), ach)) == NO_SYMBOL || id == NULL_SYMBOL_ID)
{
ODS("cannot find the symbol %c, skip!\n", c);
continue;
}
pbuf[len++] = (SYMBOL) id;
}
pbuf[len] = NULL_SYMBOL_ID;
if (len >= SP_MAX_WORD_LENGTH || len <= 0)
{
// fill in bogus pron below
}
else if (len == 1)
{
LTS_SYMTAB *tab = l_forest->symbols;
char *p = id_to_symbol(&(tab[INPUT]), pbuf[0]);
int c = tolower(p[0]);
if (c >= 'a' && c <= 'z')
{
assign_a_fixed_pron(&(l_forest->out), l_forest->single_letter_pron[c - 'a'], word);
}
}
else if (!hasvowel)
{
assign_a_spelling_pron(&(l_forest->out), l_forest->single_letter_pron, word);
}
else
{
if (allcapital)
{
assign_a_spelling_pron(&(l_forest->out), l_forest->single_letter_pron, word);
}
pres = gen_one_output(l_forest, len, pbuf, 0, pbuf, DEFAULT_PRUNE);
if (!pres)
{
return E_OUTOFMEMORY;
}
lts_fill_out_buffer(l_forest, pres, word);
}
if (l_forest->out.num_prons == 0)
{
hr = S_FALSE;
assign_a_fixed_pron(&(l_forest->out), l_forest->bogus_pron, word);
}
*ppLtsOutput = &(l_forest->out);
SPDBG_RETURN(hr);
} /* LtscartGetPron */
LTS_FOREST *LtscartReadData (LCID lcid, PBYTE map_addr)
{
SPDBG_FUNC("LtscartReadData");
int i;
LTS_FOREST *l_forest;
LTS_SYMTAB *tab;
LTS_FEATURE *feat;
int output = 0;
l_forest = (LTS_FOREST *) calloc(1, sizeof(LTS_FOREST));
if (!l_forest)
{
return NULL;
}
if (lcid == 1033)
{
l_forest->bogus_pron = bogus_pron_1033;
l_forest->single_letter_pron = single_letter_pron_1033;
}
else if (lcid == 1041)
{
l_forest->bogus_pron = bogus_pron_1041;
l_forest->single_letter_pron = single_letter_pron_1041;
}
else
{
return NULL;
}
//read in the symbol table
l_forest->symbols = (LTS_SYMTAB *) calloc(2, sizeof(LTS_SYMTAB));
if (!l_forest->symbols)
{
return NULL;
}
tab = &(l_forest->symbols[INPUT]);
CopyMemory(&(tab->n_symbols), map_addr + output, sizeof(int));
output += sizeof(int);
tab->sym_idx = (int *)(map_addr + output);
output += tab->n_symbols * sizeof(int);
CopyMemory(&(tab->n_bytes), map_addr + output, sizeof(int));
output += sizeof(int);
tab->storage = (char*)(map_addr + output);
output += tab->n_bytes * sizeof(char);
tab = &(l_forest->symbols[OUTPUT]);
CopyMemory(&(tab->n_symbols), map_addr + output, sizeof(int));
output += sizeof(int);
tab->sym_idx = (int*)(map_addr + output);
output += tab->n_symbols * sizeof(int);
CopyMemory(&(tab->n_bytes), map_addr + output, sizeof(int));
output += sizeof(int);
tab->storage = (char*)(map_addr + output);
output += tab->n_bytes * sizeof(char);
// read in the feature vector
l_forest->features = (LTS_FEATURE *) calloc(2, sizeof(LTS_FEATURE));
if (!l_forest->features)
{
return NULL;
}
feat = &(l_forest->features[INPUT]);
CopyMemory(&(feat->n_feat), map_addr + output, sizeof(int));
output += sizeof(int);
CopyMemory(&(feat->dim), map_addr + output, sizeof(int));
output += sizeof(int);
feat->feature = (int **) calloc(feat->n_feat, sizeof(int *));
if (!feat->feature)
{
return NULL;
}
for (i = 0; i < feat->n_feat; i++)
{
feat->feature[i] = (int*)(map_addr + output);
output += feat->dim * sizeof(int);
}
feat = &(l_forest->features[OUTPUT]);
CopyMemory(&(feat->n_feat), map_addr + output, sizeof(int));
output += sizeof(int);
CopyMemory(&(feat->dim), map_addr + output, sizeof(int));
output += sizeof(int);
feat->feature = (int **) calloc(feat->n_feat, sizeof(int *));
if (!feat->feature)
{
return NULL;
}
for (i = 0; i < feat->n_feat; i++)
{
feat->feature[i] = (int*)(map_addr + output);
output += feat->dim * sizeof(int);
}
/*
* read in the tree
*/
l_forest->tree = (LTS_TREE **) calloc(l_forest->symbols[INPUT].n_symbols,
sizeof(LTS_TREE *));
if (!l_forest->tree)
{
return NULL;
}
for (i = 1; i < l_forest->symbols[INPUT].n_symbols; i++)
{
LTS_TREE *l_root;
l_forest->tree[i] = l_root = (LTS_TREE *) calloc(1, sizeof(LTS_TREE));
if (!l_root)
{
return NULL;
}
CopyMemory(&(l_root->n_nodes), map_addr + output, sizeof(int));
output += sizeof(int);
l_root->nodes = (LTS_NODE*)(map_addr + output);
output += l_root->n_nodes * sizeof(LTS_NODE);
CopyMemory(&(l_root->size_dist), map_addr + output, sizeof(int));
output += sizeof(int);
l_root->p_dist = (LTS_DIST*)(map_addr + output);
output += l_root->size_dist * sizeof(char);
CopyMemory(&(l_root->size_prod), map_addr + output, sizeof(int));
output += sizeof(int);
if (l_root->size_prod > 0)
{
l_root->p_prod = (LTS_PROD*)(map_addr + output);
output += l_root->size_prod * sizeof(char);
}
}
return l_forest;
} // LTS_FOREST *LtscartReadData(char *forest_image, HANDLE *hFile1,
void LtscartFreeData(LTS_FOREST *l_forest)
{
SPDBG_FUNC("LtscartFreeData");
for (int i = 1; i < l_forest->symbols[INPUT].n_symbols; i++)
{
free(l_forest->tree[i]);
}
free(l_forest->tree);
free(l_forest->features[INPUT].feature);
free(l_forest->features[OUTPUT].feature);
free(l_forest->features);
free(l_forest->symbols);
free(l_forest);
} // void LtscartFreeData(LTS_FOREST *l_forest, HANDLE m_hFile,
Reference in a new issue Copy permalink