3195 lines
83 KiB
C++
3195 lines
83 KiB
C++
////////////////////////////////////////////////////////////////////////////////
|
|
//
|
|
// Filename : Tokenizer.cpp
|
|
// Purpose : Tokenizer implementation
|
|
//
|
|
// Project : WordBreakers
|
|
// Component: English word breaker
|
|
//
|
|
// Author : yairh
|
|
//
|
|
// Log:
|
|
//
|
|
// Jan 06 2000 yairh creation
|
|
// Apr 04 2000 dovh on behalf of dlee - Fix CTokenizer::OutputClitics
|
|
// to avoid PutWord of length 0 (leads to multiple PutWord at
|
|
// same location (duplicate keys), and index corruption!
|
|
// Example: :...'s :...'s (. stands for junk character)
|
|
// Apr 05 2000 dovh - Fixed two problematic debug / tracer buffer size
|
|
// problems. (Related to Bug 15449).
|
|
// May 07 2000 dovh - USE_WS_SENTINEL algorithm in BreakText
|
|
// May 11 2000 dovh - Simplify VerifyMisc test.
|
|
// Nov 11 2000 dovh - Special underscore treatment
|
|
// Add AddBackUnderscores '_' + alphanumeric treatment.
|
|
//
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
#include "base.h"
|
|
#include "Tokenizer.h"
|
|
#include "PropArray.h"
|
|
#include "excption.h"
|
|
#include "formats.h"
|
|
|
|
DECLARE_TRIE_SENTINEL;
|
|
CWbToUpper g_WbToUpper;
|
|
|
|
CAutoClassPointer<CPropArray> g_pPropArray;
|
|
|
|
CTokenizer::CTokenizer(
|
|
TEXT_SOURCE* pTxtSource,
|
|
IWordSink * pWordSink,
|
|
IPhraseSink * pPhraseSink,
|
|
LCID lcid,
|
|
BOOL bQueryTime,
|
|
ULONG ulMaxTokenSize) :
|
|
m_pTxtSource(pTxtSource),
|
|
m_apWordSink(pWordSink),
|
|
m_apPhraseSink(pPhraseSink),
|
|
m_Lcid(lcid),
|
|
m_bQueryTime(bQueryTime),
|
|
m_bNoMoreTxt(false),
|
|
m_Token(ulMaxTokenSize),
|
|
m_bWhiteSpaceGuarranteed(false)
|
|
{
|
|
m_ulMaxTokenSize = min(ulMaxTokenSize, TOKENIZER_MAXBUFFERLIMIT);
|
|
|
|
m_apLangSupport = new CLangSupport(lcid);
|
|
|
|
m_pCurToken = &m_Token;
|
|
|
|
if (pTxtSource->iEnd > pTxtSource->iCur)
|
|
{
|
|
CalculateUpdateEndOfBuffer();
|
|
}
|
|
else
|
|
{
|
|
m_ulUpdatedEndOfBuffer = pTxtSource->iEnd;
|
|
}
|
|
}
|
|
|
|
|
|
void CTokenizer::BreakText()
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizer,
|
|
("CTokenizer::BreakText()"));
|
|
|
|
|
|
WCHAR wch;
|
|
ULONGLONG ullflags(PROP_DEFAULT);
|
|
|
|
//
|
|
// USE_WS_SENTINEL Algorithm:
|
|
//
|
|
|
|
HRESULT hr = S_OK;
|
|
|
|
if (m_pTxtSource->iCur >= m_ulUpdatedEndOfBuffer)
|
|
{
|
|
|
|
hr = FillBuffer();
|
|
|
|
}
|
|
|
|
while ( SUCCEEDED(hr) )
|
|
{
|
|
if ( m_bWhiteSpaceGuarranteed )
|
|
{
|
|
while (true)
|
|
{
|
|
wch = m_pTxtSource->awcBuffer[m_pTxtSource->iCur];
|
|
|
|
ullflags = (GET_PROP(wch).m_ulFlag);
|
|
|
|
if (ullflags & PROP_WS)
|
|
{
|
|
if (m_pCurToken->IsNotEmpty())
|
|
{
|
|
ProcessToken();
|
|
}
|
|
m_pTxtSource->iCur++;
|
|
|
|
if (m_pTxtSource->iCur >= m_ulUpdatedEndOfBuffer)
|
|
{
|
|
hr = FillBuffer();
|
|
break;
|
|
|
|
}
|
|
continue;
|
|
|
|
}
|
|
|
|
//
|
|
// The following lines are inline expenstion of what
|
|
// used to be CToken::RecordChar:
|
|
//
|
|
|
|
Assert(m_pCurToken->m_ulBufPos < m_ulMaxTokenSize);
|
|
m_pCurToken->m_awchBuf[m_pCurToken->m_ulBufPos] = wch;
|
|
m_pCurToken->m_ulBufPos++;
|
|
m_pCurToken->m_State.m_Properties.m_ulFlag |= ullflags;
|
|
m_pTxtSource->iCur++;
|
|
|
|
} // while
|
|
}
|
|
else
|
|
{
|
|
while (true)
|
|
{
|
|
if (m_pTxtSource->iCur >= m_ulUpdatedEndOfBuffer)
|
|
{
|
|
Assert(m_pTxtSource->iCur == m_ulUpdatedEndOfBuffer);
|
|
|
|
//
|
|
// before we switch between buffers if the current token is not empty we
|
|
// need to proccess it. m_ulUpdatedEndOfBuffer always points to a breaker character
|
|
// (usually it is a WS) thus no token can start at a certain buffer and end in the
|
|
// proceeding buffer.
|
|
//
|
|
|
|
if (m_pCurToken->IsNotEmpty())
|
|
{
|
|
ProcessToken();
|
|
}
|
|
|
|
hr = FillBuffer();
|
|
if (FAILED(hr))
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
wch = m_pTxtSource->awcBuffer[m_pTxtSource->iCur];
|
|
|
|
ULONGLONG ullflags(GET_PROP(wch).m_ulFlag);
|
|
|
|
if (ullflags & PROP_WS)
|
|
{
|
|
if (m_pCurToken->IsNotEmpty())
|
|
{
|
|
ProcessToken();
|
|
}
|
|
m_pTxtSource->iCur++;
|
|
continue;
|
|
}
|
|
|
|
//
|
|
// the following lines are inline expenstion of what used to be CToken::RecordChar.
|
|
//
|
|
|
|
Assert(m_pCurToken->m_ulBufPos < m_ulMaxTokenSize);
|
|
m_pCurToken->m_awchBuf[m_pCurToken->m_ulBufPos] = wch;
|
|
m_pCurToken->m_ulBufPos++;
|
|
m_pCurToken->m_State.m_Properties.m_ulFlag |= ullflags;
|
|
m_pTxtSource->iCur++;
|
|
|
|
} // while
|
|
|
|
} // if
|
|
|
|
} // while ( !FAILED(hr) )
|
|
|
|
} // CTokenizer::BreakText
|
|
|
|
void CTokenizer::ProcessToken()
|
|
{
|
|
ULONG ulOffset;
|
|
|
|
if (m_pTxtSource->iCur < m_pCurToken->m_ulBufPos)
|
|
{
|
|
Trace(
|
|
elWarning,
|
|
s_tagTokenizer,
|
|
("CTokenizer::ProcessToken() wrong offset calculation"));
|
|
|
|
//
|
|
// BUGBUG need to understand why we got to this place.
|
|
//
|
|
|
|
ulOffset = m_pCurToken->m_ulBufPos + 1;
|
|
}
|
|
else if (m_pTxtSource->iCur == m_pCurToken->m_ulBufPos)
|
|
{
|
|
ulOffset = m_pCurToken->m_ulBufPos;
|
|
}
|
|
else
|
|
{
|
|
ulOffset = m_pTxtSource->iCur;
|
|
}
|
|
|
|
m_pCurToken->MarkEndToken(ulOffset);
|
|
#ifdef DEBUG
|
|
TraceToken();
|
|
#endif
|
|
|
|
//
|
|
// simple token.
|
|
//
|
|
if (IS_PROP_SIMPLE(m_pCurToken->m_State.m_Properties))
|
|
{
|
|
OutputSimpleToken(
|
|
m_pCurToken->m_State,
|
|
&g_EmptyClitics);
|
|
|
|
}
|
|
else
|
|
{
|
|
ProcessTokenInternal();
|
|
}
|
|
|
|
if (m_pCurToken->m_fHasEos)
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("EOS"));
|
|
|
|
HRESULT hr;
|
|
hr = m_apWordSink->PutBreak(WORDREP_BREAK_EOS);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
m_pCurToken->Clear();
|
|
}
|
|
|
|
void CTokenizer::ProcessTokenInternal()
|
|
{
|
|
|
|
do
|
|
{
|
|
|
|
//
|
|
// url
|
|
//
|
|
|
|
if (HAS_PROP_SLASH(m_pCurToken->m_State.m_Properties) &&
|
|
HAS_PROP_COLON(m_pCurToken->m_State.m_Properties) &&
|
|
HAS_PROP_ALPHA(m_pCurToken->m_State.m_Properties))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to be <alpha>:// url",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
if (VerifyAlphaUrl())
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (HAS_PROP_PERIOD(m_pCurToken->m_State.m_Properties) &&
|
|
HAS_PROP_W(m_pCurToken->m_State.m_Properties))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to be www. url",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
if (VerifyWwwUrl())
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
//
|
|
// Acronym
|
|
//
|
|
|
|
if (HAS_PROP_PERIOD(m_pCurToken->m_State.m_Properties) &&
|
|
HAS_PROP_UPPER_CASE(m_pCurToken->m_State.m_Properties))
|
|
{
|
|
if (!HAS_PROP_LOWER_CASE(m_pCurToken->m_State.m_Properties) ||
|
|
HAS_PROP_APOSTROPHE(m_pCurToken->m_State.m_Properties))
|
|
{
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to be an acronym",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
if (VerifyAcronym())
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
//
|
|
// Abbreviation
|
|
//
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to be an abbreviation",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
|
|
if (VerifyAbbreviation())
|
|
{
|
|
break;
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to be a special abbreviation",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
if (VerifySpecialAbbreviation())
|
|
{
|
|
break;
|
|
}
|
|
|
|
}
|
|
|
|
//
|
|
// Hyphenation
|
|
//
|
|
if (HAS_PROP_DASH(m_pCurToken->m_State.m_Properties) &&
|
|
HAS_PROP_ALPHA(m_pCurToken->m_State.m_Properties))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to have a hyphenation",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
if (VerifyHyphenation())
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
//
|
|
// (s) parenthesis
|
|
//
|
|
|
|
if (HAS_PROP_LEFT_PAREN(m_pCurToken->m_State.m_Properties) &&
|
|
HAS_PROP_RIGHT_PAREN(m_pCurToken->m_State.m_Properties) &&
|
|
HAS_PROP_ALPHA(m_pCurToken->m_State.m_Properties))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to have a (s) Parenthesis",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
if (VerifyParens())
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
//
|
|
// Currency
|
|
//
|
|
if (HAS_PROP_CURRENCY(m_pCurToken->m_State.m_Properties) &&
|
|
HAS_PROP_NUMBER(m_pCurToken->m_State.m_Properties))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to be a currency",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
if (VerifyCurrency())
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
//
|
|
// Numbers / time / dates
|
|
//
|
|
|
|
if (HAS_PROP_NUMBER(m_pCurToken->m_State.m_Properties))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to be a number or a time or a date",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
if (VerifyNumberOrTimeOrDate())
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
//
|
|
// commersial signs
|
|
//
|
|
if (TEST_PROP(m_pCurToken->m_State.m_Properties, PROP_COMMERSIAL_SIGN) &&
|
|
HAS_PROP_ALPHA(m_pCurToken->m_State.m_Properties))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to have a commesial sign",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
if (VerifyCommersialSign())
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
//
|
|
// Misc - C++, J++, A+, A- .. C#
|
|
//
|
|
|
|
if ( TEST_PROP(m_pCurToken->m_State.m_Properties, (PROP_MINUS|PROP_PLUS|PROP_POUND)) &&
|
|
HAS_PROP_ALPHA(m_pCurToken->m_State.m_Properties) )
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to belong to the misc list",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
|
|
if (VerifyMisc())
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
//
|
|
// default
|
|
//
|
|
|
|
ProcessDefault();
|
|
|
|
} while (false);
|
|
|
|
}
|
|
|
|
#ifdef DEBUG
|
|
void CTokenizer::TraceToken()
|
|
{
|
|
WCHAR buf[MAX_NUM_PROP+1];
|
|
|
|
size_t bufLen = wcslen(TRACE_CHAR);
|
|
Assert(bufLen < MAX_NUM_PROP + 1);
|
|
buf[bufLen] = L'\0';
|
|
|
|
for(int i=0; i<bufLen; i++)
|
|
{
|
|
if(TEST_PROP(m_pCurToken->m_State.m_Properties, (1<<i)))
|
|
{
|
|
buf[i] = TRACE_CHAR[i];
|
|
}
|
|
else
|
|
{
|
|
buf[i] = L'_';
|
|
}
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerTrace,
|
|
("[%S] - %*.*S",
|
|
buf,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
|
|
}
|
|
#endif // DEBUG
|
|
|
|
bool CTokenizer::VerifyAlphaUrl()
|
|
{
|
|
//
|
|
// looking for <alpha>:// pattern
|
|
//
|
|
|
|
CTokenState State(m_pCurToken->m_State);
|
|
|
|
ULONG ul = State.m_ulStart;
|
|
|
|
if (!HAS_PROP_ALPHA(GET_PROP(State.m_pwcsToken[ul])))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
while (HAS_PROP_EXTENDED_ALPHA(GET_PROP(State.m_pwcsToken[ul])))
|
|
{
|
|
ul++;
|
|
}
|
|
|
|
if (!(HAS_PROP_COLON(GET_PROP(State.m_pwcsToken[ul]))))
|
|
{
|
|
return false;
|
|
}
|
|
ul++;
|
|
|
|
if (!(HAS_PROP_SLASH(GET_PROP(State.m_pwcsToken[ul]))))
|
|
{
|
|
return false;
|
|
}
|
|
ul++;
|
|
|
|
if (!(HAS_PROP_SLASH(GET_PROP(State.m_pwcsToken[ul]))))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is an <alpha>:// url",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
}
|
|
|
|
OutputUrl(State);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool CTokenizer::VerifyWwwUrl()
|
|
{
|
|
CTokenState State(m_pCurToken->m_State);
|
|
|
|
if (State.m_ulEnd - State.m_ulStart <= 4)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (0 != _wcsnicmp(State.m_pwcsToken + State.m_ulStart, L"www.", 4))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is a www. url",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
OutputUrl(State);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool CTokenizer::VerifyAcronym()
|
|
{
|
|
//
|
|
// looking for I.B.M or I.B.M. or A.B.CC but not A.B.CC.
|
|
//
|
|
|
|
CTokenState State(m_pCurToken->m_State);
|
|
|
|
CPropFlag AbbPuctTail(ACRONYM_PUNCT_TAIL);
|
|
CPropFlag AbbPuctHead(ACRONYM_PUNCT_HEAD);
|
|
bool fNeedToRemoveEos = true;
|
|
|
|
if (TEST_PROP(State.m_Properties, (ACRONYM_PUNCT_TAIL | ACRONYM_PUNCT_HEAD)))
|
|
{
|
|
if (TEST_PROP(GET_PROP(State.m_pwcsToken[State.m_ulEnd- 1]), ABBREVIATION_EOS))
|
|
{
|
|
fNeedToRemoveEos = false;
|
|
}
|
|
|
|
ULONG ulCharRemoved = m_pCurToken->RemoveTailPunct(AbbPuctTail, State);
|
|
ulCharRemoved += m_pCurToken->RemoveHeadPunct(AbbPuctHead, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
const CCliticsTerm* pCliticsTerm;
|
|
pCliticsTerm = VerifyClitics(State);
|
|
|
|
ULONG ulEnd = State.m_ulEnd;
|
|
ULONG ulCur = State.m_ulStart;
|
|
|
|
if (pCliticsTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
ulCur += pCliticsTerm->ulLen;
|
|
}
|
|
else if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
ulEnd -= pCliticsTerm->ulLen;
|
|
}
|
|
|
|
//
|
|
// finding the last period
|
|
//
|
|
while ((ulEnd > ulCur) &&
|
|
HAS_PROP_UPPER_CASE(GET_PROP(State.m_pwcsToken[ulEnd- 1])))
|
|
{
|
|
ulEnd--;
|
|
}
|
|
|
|
if ((ulEnd == ulCur) ||
|
|
!HAS_PROP_PERIOD(GET_PROP(State.m_pwcsToken[ulEnd- 1])))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
ULONG ulCounter = 0;
|
|
|
|
while (ulCur < ulEnd)
|
|
{
|
|
if (ulCounter%2 == 0)
|
|
{
|
|
if (!HAS_PROP_UPPER_CASE(GET_PROP(State.m_pwcsToken[ulCur])))
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (!HAS_PROP_PERIOD(GET_PROP(State.m_pwcsToken[ulCur])))
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
ulCur++;
|
|
ulCounter++;
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is an acronym",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
if (fNeedToRemoveEos && (pCliticsTerm->ulOp != TAIL_MATCH_TRUNCATE))
|
|
{
|
|
m_pCurToken->m_fHasEos = false;
|
|
}
|
|
OutputAcronym(State, pCliticsTerm);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool CTokenizer::VerifyAbbreviation()
|
|
{
|
|
//
|
|
// looking for Sr. Jr.
|
|
// we define abbreviation as a pattern with 2 letters ending with a dot and the first letter
|
|
// is a capital one
|
|
//
|
|
|
|
CTokenState State(m_pCurToken->m_State);
|
|
CPropFlag AbbPuctTail(ABBREVIATION_PUNCT_TAIL);
|
|
CPropFlag AbbPuctHead(ABBREVIATION_PUNCT_HEAD);
|
|
bool fNeedToRemoveEos = true;
|
|
|
|
if (TEST_PROP(State.m_Properties, (ABBREVIATION_PUNCT_TAIL | ABBREVIATION_PUNCT_HEAD)))
|
|
{
|
|
if (TEST_PROP(GET_PROP(State.m_pwcsToken[State.m_ulEnd- 1]), ABBREVIATION_EOS))
|
|
{
|
|
fNeedToRemoveEos = false;
|
|
}
|
|
|
|
ULONG ulCharRemoved = m_pCurToken->RemoveTailPunct(AbbPuctTail, State);
|
|
ulCharRemoved += m_pCurToken->RemoveHeadPunct(AbbPuctHead, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
if ((State.m_ulEnd - State.m_ulStart) != 3)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (!HAS_PROP_UPPER_CASE(GET_PROP(State.m_pwcsToken[State.m_ulStart])))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (!HAS_PROP_EXTENDED_ALPHA(GET_PROP(State.m_pwcsToken[State.m_ulStart + 1])))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (!HAS_PROP_PERIOD(GET_PROP(State.m_pwcsToken[State.m_ulStart + 2])))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is an abbreviation",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
|
|
if (fNeedToRemoveEos)
|
|
{
|
|
m_pCurToken->m_fHasEos = false;
|
|
}
|
|
|
|
OutputAbbreviation(State);
|
|
return true;
|
|
|
|
}
|
|
|
|
bool CTokenizer::VerifySpecialAbbreviation()
|
|
{
|
|
CTokenState State(m_pCurToken->m_State);
|
|
CPropFlag AbbPuctTail(SPECIAL_ABBREVIATION_PUNCT_TAIL);
|
|
CPropFlag AbbPuctHead(SPECIAL_ABBREVIATION_PUNCT_HEAD);
|
|
|
|
if (TEST_PROP(State.m_Properties, (SPECIAL_ABBREVIATION_PUNCT_TAIL | SPECIAL_ABBREVIATION_PUNCT_HEAD)))
|
|
{
|
|
ULONG ulCharRemoved = m_pCurToken->RemoveTailPunct(AbbPuctTail, State);
|
|
ulCharRemoved += m_pCurToken->RemoveHeadPunct(AbbPuctHead, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
|
|
if (!HAS_PROP_PERIOD(State.m_Properties))
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
const CCliticsTerm* pCliticsTerm;
|
|
pCliticsTerm = VerifyClitics(State);
|
|
|
|
ULONG ulAddToStart = 0;
|
|
ULONG ulDecFromEnd = 0;
|
|
|
|
if (pCliticsTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
ulAddToStart = pCliticsTerm->ulLen;
|
|
}
|
|
else if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
ulDecFromEnd = pCliticsTerm->ulLen;
|
|
}
|
|
|
|
|
|
CAbbTerm* pTerm;
|
|
short sResCount = 0;
|
|
DictStatus status;
|
|
|
|
CSpecialAbbreviationSet* pAbbSet = m_apLangSupport->GetAbbSet();
|
|
status = pAbbSet->m_trieAbb.trie_Find(
|
|
State.m_pwcsToken + State.m_ulStart + ulAddToStart,
|
|
TRIE_LONGEST_MATCH | TRIE_IGNORECASE,
|
|
1,
|
|
&pTerm,
|
|
&sResCount);
|
|
|
|
if (sResCount &&
|
|
(pTerm->ulAbbLen == (State.m_ulEnd - State.m_ulStart - ulAddToStart - ulDecFromEnd)))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is an abbreviation",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
OutputSpecialAbbreviation(State, pTerm, pCliticsTerm);
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool CTokenizer::VerifyMisc()
|
|
{
|
|
CTokenState State(m_pCurToken->m_State);
|
|
CPropFlag MiscPuctTail(MISC_PUNCT_TAIL);
|
|
CPropFlag MiscPuctHead(MISC_PUNCT_HEAD);
|
|
|
|
if (TEST_PROP(State.m_Properties, (MISC_PUNCT_TAIL | MISC_PUNCT_HEAD)))
|
|
{
|
|
ULONG ulCharRemoved = m_pCurToken->RemoveTailPunct(MiscPuctTail, State);
|
|
ulCharRemoved += m_pCurToken->RemoveHeadPunct(MiscPuctHead, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
const CCliticsTerm* pCliticsTerm;
|
|
pCliticsTerm = VerifyClitics(State);
|
|
|
|
ULONG ulAddToStart = 0;
|
|
ULONG ulDecFromEnd = 0;
|
|
|
|
if (pCliticsTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
ulAddToStart = pCliticsTerm->ulLen;
|
|
}
|
|
else if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
ulDecFromEnd = pCliticsTerm->ulLen;
|
|
}
|
|
|
|
bool bPatternContainOnlyUpperCase = true;
|
|
ULONG ulSuffixSize = 0;
|
|
|
|
if (TEST_PROP(State.m_Properties, PROP_POUND))
|
|
{
|
|
//
|
|
// look for A# C#
|
|
//
|
|
|
|
ULONG ulEnd = State.m_ulEnd - ulDecFromEnd;
|
|
ULONG ulStart = State.m_ulStart + ulAddToStart;
|
|
if (ulEnd - ulStart != 2)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (!TEST_PROP(GET_PROP(State.m_pwcsToken[ulEnd - 1]), PROP_POUND))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (!TEST_PROP(GET_PROP(State.m_pwcsToken[ulStart]), PROP_UPPER_CASE))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
ulSuffixSize = 1;
|
|
}
|
|
else
|
|
{
|
|
//
|
|
// look for C++ COM+ ...
|
|
//
|
|
|
|
ULONG ul = State.m_ulEnd - ulDecFromEnd - 1;
|
|
while ((int)ul >= (int)(State.m_ulStart + ulAddToStart))
|
|
{
|
|
if (!TEST_PROP(GET_PROP(State.m_pwcsToken[ul]), PROP_PLUS | PROP_MINUS))
|
|
{
|
|
break;
|
|
}
|
|
ulSuffixSize++;
|
|
ul--;
|
|
}
|
|
|
|
if (ulSuffixSize > 2)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
while ((int)ul >= (int)(State.m_ulStart + ulAddToStart))
|
|
{
|
|
CPropFlag prop(GET_PROP(State.m_pwcsToken[ul]));
|
|
if (!HAS_PROP_EXTENDED_ALPHA(prop))
|
|
{
|
|
return false;
|
|
}
|
|
if (!TEST_PROP(prop, PROP_UPPER_CASE))
|
|
{
|
|
bPatternContainOnlyUpperCase = false;
|
|
}
|
|
|
|
ul--;
|
|
}
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is detected",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
OutputMisc(
|
|
State,
|
|
bPatternContainOnlyUpperCase,
|
|
ulSuffixSize,
|
|
pCliticsTerm);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
bool CTokenizer::VerifyHyphenation()
|
|
{
|
|
//
|
|
// looking for data-base
|
|
//
|
|
|
|
CPropFlag PunctHead(HYPHENATION_PUNCT_HEAD);
|
|
CPropFlag PunctTail(HYPHENATION_PUNCT_TAIL);
|
|
CTokenState State(m_pCurToken->m_State);
|
|
|
|
if (TEST_PROP(State.m_Properties, (HYPHENATION_PUNCT_HEAD | HYPHENATION_PUNCT_TAIL)))
|
|
{
|
|
ULONG ulCharRemoved;
|
|
ulCharRemoved = m_pCurToken->RemoveHeadPunct(PunctHead, State);
|
|
ulCharRemoved += m_pCurToken->RemoveTailPunct(PunctTail, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
if (!HAS_PROP_DASH(State.m_Properties))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
const CCliticsTerm* pCliticsTerm;
|
|
pCliticsTerm = VerifyClitics(State);
|
|
|
|
ULONG ulAddToStart = 0;
|
|
ULONG ulDecFromEnd = 0;
|
|
|
|
if (pCliticsTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
ulAddToStart = pCliticsTerm->ulLen;
|
|
}
|
|
else if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
ulDecFromEnd = pCliticsTerm->ulLen;
|
|
}
|
|
|
|
|
|
ULONG ulCur = State.m_ulStart + ulAddToStart;
|
|
ULONG ulEnd = State.m_ulEnd - ulDecFromEnd;
|
|
|
|
bool bReadAlpha = false;
|
|
|
|
do
|
|
{
|
|
while (ulCur < ulEnd)
|
|
{
|
|
if (HAS_PROP_EXTENDED_ALPHA(GET_PROP(m_pCurToken->m_State.m_pwcsToken[ulCur])))
|
|
{
|
|
ulCur++;
|
|
bReadAlpha = true;
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (!bReadAlpha)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (ulCur < ulEnd)
|
|
{
|
|
if (!HAS_PROP_DASH(GET_PROP(m_pCurToken->m_State.m_pwcsToken[ulCur])))
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
|
|
ulCur++;
|
|
bReadAlpha = false;
|
|
}
|
|
while (ulCur < ulEnd);
|
|
|
|
if (!bReadAlpha)
|
|
{
|
|
//
|
|
// last characters where not alpha ex. free-
|
|
//
|
|
return false;
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is an hyphenation",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
OutputHyphenation(State, pCliticsTerm);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool CTokenizer::VerifyParens()
|
|
{
|
|
CPropFlag PunctTail(PAREN_PUNCT_TAIL);
|
|
CPropFlag PunctHead(PAREN_PUNCT_HEAD);
|
|
|
|
CTokenState State(m_pCurToken->m_State);
|
|
|
|
if (TEST_PROP(State.m_Properties, (PAREN_PUNCT_TAIL | PAREN_PUNCT_HEAD)))
|
|
{
|
|
ULONG ulCharRemoved;
|
|
ulCharRemoved = m_pCurToken->RemoveTailPunct(PunctTail, State);
|
|
ulCharRemoved += m_pCurToken->RemoveHeadPunct(PunctHead, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
//
|
|
// looking for (s)
|
|
//
|
|
|
|
if ((State.m_ulEnd - State.m_ulStart) < 4)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (0 != wcsncmp(State.m_pwcsToken + State.m_ulEnd - 3, L"(s)", 3))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
for (ULONG ul = State.m_ulStart; ul < State.m_ulEnd - 3; ul++)
|
|
{
|
|
if (!HAS_PROP_EXTENDED_ALPHA(GET_PROP(State.m_pwcsToken[ul])))
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S has (s) parenthesis",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
OutputParens(State);
|
|
|
|
return true;
|
|
}
|
|
|
|
const CCliticsTerm* CTokenizer::VerifyClitics(CTokenState& S)
|
|
{
|
|
if (TEST_PROP(GET_PROP(S.m_pwcsToken[S.m_ulStart]), PROP_APOSTROPHE))
|
|
{
|
|
S.m_ulStart++;
|
|
|
|
if ((TEST_PROP(GET_PROP(S.m_pwcsToken[S.m_ulEnd - 1]), PROP_APOSTROPHE)) &&
|
|
(S.m_ulEnd > S.m_ulStart))
|
|
{
|
|
S.m_ulEnd--;
|
|
}
|
|
|
|
m_pCurToken->ComputeStateProperties(S);
|
|
}
|
|
|
|
if (!(HAS_PROP_APOSTROPHE(S.m_Properties)))
|
|
{
|
|
return &g_EmptyClitics;
|
|
}
|
|
|
|
CPropFlag PunctTail(CLITICS_PUNC_TAIL);
|
|
CPropFlag PunctHead(CLITICS_PUNCT_HEAD);
|
|
|
|
CTokenState State(S);
|
|
|
|
if (TEST_PROP(State.m_Properties, (CLITICS_PUNC_TAIL | CLITICS_PUNCT_HEAD)))
|
|
{
|
|
ULONG ulCharRemoved;
|
|
ulCharRemoved = m_pCurToken->RemoveTailPunct(PunctTail, State);
|
|
ulCharRemoved += m_pCurToken->RemoveHeadPunct(PunctHead, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to have an apostophe",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
|
|
|
|
ULONG ulApostrophePos = -1;
|
|
ULONG ulCur;
|
|
for (ulCur = State.m_ulStart; ulCur < State.m_ulEnd ; ulCur++)
|
|
{
|
|
|
|
if (TEST_PROP(GET_PROP(State.m_pwcsToken[ulCur]), PROP_APOSTROPHE))
|
|
{
|
|
if ((-1 != ulApostrophePos) || (State.m_ulStart == ulCur))
|
|
{
|
|
//
|
|
// this is not the first \' this is not a valid clitics
|
|
// or the term start with a new apostrophe
|
|
//
|
|
return &g_EmptyClitics;
|
|
}
|
|
ulApostrophePos = ulCur;
|
|
//
|
|
// replace the apostrophe with an ascii apostrophe.
|
|
//
|
|
State.m_pwcsToken[ulCur] = L'\'';
|
|
continue;
|
|
}
|
|
}
|
|
|
|
//
|
|
// looking for xxxxs'
|
|
//
|
|
if ((ulApostrophePos == State.m_ulEnd - 1) &&
|
|
(State.m_pwcsToken[ulApostrophePos - 1] == L's'))
|
|
{
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S has a s' clitcs",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
S = State;
|
|
return &g_SClitics;
|
|
|
|
}
|
|
|
|
//
|
|
// looking for tail clitics like xxx's
|
|
//
|
|
|
|
DictStatus status;
|
|
|
|
CCliticsTerm* pTerm;
|
|
short sResCount = 0;
|
|
|
|
if (ulCur > State.m_ulStart)
|
|
{
|
|
status = g_pClitics->m_trieClitics.trie_Find(
|
|
State.m_pwcsToken + ulApostrophePos,
|
|
TRIE_LONGEST_MATCH | TRIE_IGNORECASE,
|
|
1,
|
|
&pTerm,
|
|
&sResCount);
|
|
if (sResCount && pTerm->ulLen == (State.m_ulEnd - ulApostrophePos))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S has a %S clitcs",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart,
|
|
pTerm->pwcs
|
|
));
|
|
|
|
S = State;
|
|
return pTerm;
|
|
}
|
|
}
|
|
|
|
//
|
|
// looking for head clitics like l'xxxx
|
|
//
|
|
|
|
status = g_pClitics->m_trieClitics.trie_Find(
|
|
State.m_pwcsToken + State.m_ulStart,
|
|
TRIE_LONGEST_MATCH | TRIE_IGNORECASE,
|
|
1,
|
|
&pTerm,
|
|
&sResCount);
|
|
if (sResCount)
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S has a %S clitcs",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart,
|
|
pTerm->pwcs
|
|
));
|
|
|
|
S = State;
|
|
return pTerm;
|
|
}
|
|
|
|
return &g_EmptyClitics;
|
|
}
|
|
|
|
bool CTokenizer::VerifyNumberOrTimeOrDate()
|
|
{
|
|
CPropFlag PunctHead(NUM_DATE_TIME_PUNCT_HEAD);
|
|
CPropFlag PunctTail(NUM_DATE_TIME_PUNCT_TAIL);
|
|
CTokenState State(m_pCurToken->m_State);
|
|
|
|
if (TEST_PROP(State.m_Properties,
|
|
(NUM_DATE_TIME_PUNCT_HEAD | NUM_DATE_TIME_PUNCT_TAIL)))
|
|
{
|
|
ULONG ulCharRemoved;
|
|
ulCharRemoved= m_pCurToken->RemoveHeadPunct(PunctHead, State);
|
|
ulCharRemoved += m_pCurToken->RemoveTailPunct(PunctTail, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
if ((TEST_PROP(
|
|
State.m_Properties,
|
|
(GET_PROP(m_apLangSupport->GetTimeSeperator()).m_ulFlag))) ||
|
|
HAS_PROP_ALPHA(State.m_Properties))
|
|
{
|
|
//
|
|
// suspected to be time 12:33 14:22 15:22:33
|
|
// or AM/PM time format 12:22AM 13PM
|
|
//
|
|
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to be AM/PM time",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
if (VerifyTime(State))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
}
|
|
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to be a simple number",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
if (VerifyNumber(State))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
if (TEST_PROP(State.m_Properties, PROP_DATE_SEPERATOR))
|
|
{
|
|
//
|
|
// suspected to be a date 1999-05-04 or 1998/11/10 1999.05.04
|
|
//
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerSuspect,
|
|
("%*.*S suspected to be a date",
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_ulEnd - m_pCurToken->m_State.m_ulStart,
|
|
m_pCurToken->m_State.m_pwcsToken + m_pCurToken->m_State.m_ulStart
|
|
));
|
|
|
|
return VerifyDate(State);
|
|
}
|
|
|
|
|
|
return false;
|
|
}
|
|
|
|
|
|
bool CTokenizer::VerifyTime(CTokenState& S)
|
|
{
|
|
CTokenState State(S);
|
|
CPropFlag PunctHead(TIME_ADDITIONAL_PUNCT_HEAD);
|
|
CPropFlag PunctTail(TIME_ADDITIONAL_PUNCT_TAIL);
|
|
|
|
if (TEST_PROP(State.m_Properties,
|
|
(TIME_ADDITIONAL_PUNCT_HEAD | TIME_ADDITIONAL_PUNCT_TAIL)))
|
|
{
|
|
ULONG ulCharRemoved;
|
|
ulCharRemoved= m_pCurToken->RemoveHeadPunct(PunctHead, State);
|
|
ulCharRemoved += m_pCurToken->RemoveTailPunct(PunctTail, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
if ((State.m_ulEnd - State.m_ulStart) > MAX_TIME_FORMAT_LEN)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
WCHAR pwcsBuf[MAX_TIME_FORMAT_LEN + 1];
|
|
ULONG ulCur = State.m_ulStart;
|
|
WCHAR wcSeperator = 0xFFFF;
|
|
ULONG ul = 0;
|
|
|
|
//
|
|
// formatting the text to a date format
|
|
//
|
|
|
|
while (ulCur < State.m_ulEnd)
|
|
{
|
|
CPropFlag prop(GET_PROP(State.m_pwcsToken[ulCur]));
|
|
if (HAS_PROP_NUMBER(prop))
|
|
{
|
|
pwcsBuf[ul] = L'#';
|
|
}
|
|
else if (State.m_pwcsToken[ulCur] == m_apLangSupport->GetTimeSeperator())
|
|
{
|
|
if (0xFFFF == wcSeperator)
|
|
{
|
|
wcSeperator = State.m_pwcsToken[ulCur];
|
|
}
|
|
else if (wcSeperator != State.m_pwcsToken[ulCur])
|
|
{
|
|
return false;
|
|
}
|
|
pwcsBuf[ul] = L':';
|
|
}
|
|
else if (HAS_PROP_ALPHA(prop) || HAS_PROP_PERIOD(prop))
|
|
{
|
|
pwcsBuf[ul] = State.m_pwcsToken[ulCur];
|
|
}
|
|
else
|
|
{
|
|
return false;
|
|
}
|
|
|
|
ul++;
|
|
ulCur++;
|
|
}
|
|
|
|
pwcsBuf[ul] = L'\0';
|
|
|
|
CTimeTerm* pTerm;
|
|
short sResCount = 0;
|
|
DictStatus status;
|
|
|
|
status = g_pTimeFormat->m_trieTimeFormat.trie_Find(
|
|
pwcsBuf,
|
|
TRIE_LONGEST_MATCH | TRIE_IGNORECASE,
|
|
1,
|
|
&pTerm,
|
|
&sResCount);
|
|
if (!(sResCount && (pTerm->bLen == ul)))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
LONG lHour;
|
|
LONG lMin;
|
|
LONG lSec;
|
|
TimeFormat AmPm;
|
|
|
|
GetValuesFromTimeString(
|
|
pTerm,
|
|
State.m_pwcsToken + State.m_ulStart ,
|
|
&lHour,
|
|
&lMin,
|
|
&lSec,
|
|
&AmPm);
|
|
|
|
if (None == AmPm)
|
|
{
|
|
if (lHour > 24)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (lHour > 12)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (Am == AmPm)
|
|
{
|
|
if (12 == lHour)
|
|
{
|
|
lHour = 0;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (lHour < 12)
|
|
{
|
|
lHour += 12;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
if (lMin > 59)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (lSec > 59)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
WCHAR pwcsTime[9] = {L'\0',L'\0',L'\0',L'\0',L'\0',L'\0',L'\0',L'\0',L'\0',};
|
|
|
|
swprintf(pwcsTime, L"TT%02d%02d", lHour, lMin);
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is a time -> %S",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart,
|
|
pwcsTime));
|
|
|
|
OutputTime(pwcsTime, State);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool CTokenizer::VerifyDate(CTokenState& S)
|
|
{
|
|
CTokenState State(S);
|
|
CPropFlag PunctHead(DATE_ADDITIONAL_PUNCT_HEAD);
|
|
CPropFlag PunctTail(DATE_ADDITIONAL_PUNCT_TAIL);
|
|
if (TEST_PROP(State.m_Properties,
|
|
(DATE_ADDITIONAL_PUNCT_HEAD | DATE_ADDITIONAL_PUNCT_TAIL)))
|
|
{
|
|
ULONG ulCharRemoved;
|
|
ulCharRemoved= m_pCurToken->RemoveHeadPunct(PunctHead, State);
|
|
ulCharRemoved += m_pCurToken->RemoveTailPunct(PunctTail, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
|
|
WCHAR pwcsBuf[MAX_DATE_FORMAT_LEN + 1];
|
|
|
|
if (State.m_ulEnd - State.m_ulStart > MAX_DATE_FORMAT_LEN)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
ULONG ulCur = State.m_ulStart;
|
|
WCHAR wcSeperator = 0xFFFF;
|
|
ULONG ul = 0;
|
|
|
|
//
|
|
// formatting the text to a date format
|
|
//
|
|
|
|
while (ulCur < State.m_ulEnd)
|
|
{
|
|
CPropFlag prop(GET_PROP(State.m_pwcsToken[ulCur]));
|
|
if (HAS_PROP_NUMBER(prop))
|
|
{
|
|
pwcsBuf[ul] = L'#';
|
|
}
|
|
else if (HAS_PROP_PERIOD(prop) ||
|
|
HAS_PROP_DASH(prop) ||
|
|
HAS_PROP_SLASH(prop))
|
|
{
|
|
if (0xFFFF == wcSeperator)
|
|
{
|
|
wcSeperator = State.m_pwcsToken[ulCur];
|
|
}
|
|
else if (wcSeperator != State.m_pwcsToken[ulCur])
|
|
{
|
|
return false;
|
|
}
|
|
pwcsBuf[ul] = L'.';
|
|
}
|
|
else
|
|
{
|
|
return false;
|
|
}
|
|
|
|
ul++;
|
|
ulCur++;
|
|
}
|
|
|
|
pwcsBuf[ul] = L'\0';
|
|
|
|
CDateTerm* pTerm;
|
|
short sResCount = 0;
|
|
DictStatus status;
|
|
|
|
status = g_pDateFormat->m_trieDateFormat.trie_Find(
|
|
pwcsBuf,
|
|
TRIE_LONGEST_MATCH | TRIE_IGNORECASE,
|
|
1,
|
|
&pTerm,
|
|
&sResCount);
|
|
if (!(sResCount && (pTerm->bLen == ul)))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
LONG lD_M1;
|
|
LONG lD_M2;
|
|
LONG lYear;
|
|
|
|
GetValuesFromDateString(
|
|
pTerm,
|
|
State.m_pwcsToken + State.m_ulStart,
|
|
&lD_M1,
|
|
&lD_M2,
|
|
&lYear);
|
|
|
|
LONG lDay;
|
|
LONG lMonth;
|
|
|
|
//
|
|
// language dependent
|
|
//
|
|
|
|
if (m_apLangSupport->IsDayMonthOrder() ||
|
|
pTerm->bType == YYMMDD_TYPE)
|
|
{
|
|
lDay = lD_M1;
|
|
lMonth = lD_M2;
|
|
}
|
|
else
|
|
{
|
|
lDay = lD_M2;
|
|
lMonth = lD_M1;
|
|
}
|
|
|
|
if (!((lDay > 0) && (lDay <= 31)))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (!((lMonth > 0) && (lMonth <= 12)))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
|
|
WCHAR pwcsDate1[11] = { L'D', L'D', L'0', L'0', L'0', L'0', L'0', L'0', L'0', L'0', L'\0'};
|
|
WCHAR pwcsDate2[11];
|
|
bool bY2K = false;
|
|
|
|
if (lYear <= 99) // Y2k bug
|
|
{
|
|
_ltow(lYear + 1900, pwcsDate1 + 2, 10);
|
|
bY2K = true;
|
|
}
|
|
else if (lYear < 1000)
|
|
{
|
|
_ltow(lYear, pwcsDate1 + 3, 10);
|
|
}
|
|
else
|
|
{
|
|
_ltow(lYear, pwcsDate1 + 2, 10);
|
|
}
|
|
|
|
if (lMonth < 10)
|
|
{
|
|
pwcsDate1[6] = L'0';
|
|
_ltow(lMonth, pwcsDate1 + 7, 10);
|
|
}
|
|
else
|
|
{
|
|
_ltow(lMonth, pwcsDate1 + 6, 10);
|
|
}
|
|
|
|
if (lDay < 10)
|
|
{
|
|
pwcsDate1[8] = L'0';
|
|
_ltow(lDay, pwcsDate1 + 9, 10);
|
|
}
|
|
else
|
|
{
|
|
_ltow(lDay, pwcsDate1 + 8, 10);
|
|
}
|
|
|
|
if (bY2K)
|
|
{
|
|
wcscpy(pwcsDate2, pwcsDate1);
|
|
pwcsDate2[2] = L'2';
|
|
pwcsDate2[3] = L'0';
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is a date",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
if (bY2K)
|
|
{
|
|
OutputDate(pwcsDate1, pwcsDate2, State);
|
|
}
|
|
else
|
|
{
|
|
OutputDate(pwcsDate1, NULL, State);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool CTokenizer::VerifyNumber(CTokenState& S)
|
|
{
|
|
CTokenState State(S);
|
|
|
|
WCHAR pwcsNumber[TOKENIZER_MAXBUFFERLIMIT + 10];
|
|
|
|
ULONG ulOutLen;
|
|
ULONG ulOffsetToTxt;
|
|
|
|
const CCliticsTerm* pCliticsTerm;
|
|
pCliticsTerm = VerifyClitics(State);
|
|
|
|
ULONG ulAddToStart = 0;
|
|
ULONG ulDecFromEnd = 0;
|
|
|
|
if (pCliticsTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
ulAddToStart = pCliticsTerm->ulLen;
|
|
}
|
|
else if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
ulDecFromEnd = pCliticsTerm->ulLen;
|
|
}
|
|
|
|
bool fRet = CheckAndCreateNumber(
|
|
State.m_pwcsToken + State.m_ulStart + ulAddToStart,
|
|
State.m_ulEnd - State.m_ulStart - ulAddToStart - ulDecFromEnd,
|
|
pwcsNumber,
|
|
&ulOffsetToTxt,
|
|
&ulOutLen);
|
|
|
|
if (!fRet)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is a number",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
OutputNumbers(State, ulOutLen, pwcsNumber + ulOffsetToTxt, pCliticsTerm);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool CTokenizer::VerifyCurrency()
|
|
{
|
|
//
|
|
// format is either $12.22 or 12.22$
|
|
//
|
|
CPropFlag PunctHead(CURRENCY_PUNCT_HEAD);
|
|
CPropFlag PunctTail(CURRENCY_PUNCT_TAIL);
|
|
CTokenState State(m_pCurToken->m_State);
|
|
|
|
if (TEST_PROP(State.m_Properties,
|
|
(CURRENCY_PUNCT_HEAD | CURRENCY_PUNCT_TAIL)))
|
|
{
|
|
ULONG ulCharRemoved;
|
|
ulCharRemoved= m_pCurToken->RemoveHeadPunct(PunctHead, State);
|
|
ulCharRemoved += m_pCurToken->RemoveTailPunct(PunctTail, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
const CCliticsTerm* pCliticsTerm;
|
|
pCliticsTerm = VerifyClitics(State);
|
|
|
|
ULONG ulAddToStart = 0;
|
|
ULONG ulDecFromEnd = 0;
|
|
|
|
if (pCliticsTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
ulAddToStart = pCliticsTerm->ulLen;
|
|
}
|
|
else if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
ulDecFromEnd = pCliticsTerm->ulLen;
|
|
}
|
|
|
|
|
|
WCHAR wchCurrency;
|
|
WCHAR pwcsCurrency[TOKENIZER_MAXBUFFERLIMIT + 10];
|
|
WCHAR* pwcsStr = State.m_pwcsToken + State.m_ulStart;
|
|
|
|
if (HAS_PROP_CURRENCY(GET_PROP(State.m_pwcsToken[State.m_ulStart + ulAddToStart])))
|
|
{
|
|
wchCurrency = State.m_pwcsToken[State.m_ulStart + ulAddToStart];
|
|
pwcsStr += 1;
|
|
}
|
|
else if (HAS_PROP_CURRENCY(GET_PROP(State.m_pwcsToken[State.m_ulEnd - 1 - ulDecFromEnd])))
|
|
{
|
|
wchCurrency = State.m_pwcsToken[State.m_ulEnd - 1 - ulDecFromEnd];
|
|
}
|
|
else
|
|
{
|
|
return false;
|
|
}
|
|
|
|
ULONG ulOutLen;
|
|
ULONG ulOffsetToTxt;
|
|
|
|
if (false == CheckAndCreateNumber(
|
|
pwcsStr + ulAddToStart,
|
|
State.m_ulEnd - State.m_ulStart - 1 - ulAddToStart - ulDecFromEnd,
|
|
pwcsCurrency,
|
|
&ulOffsetToTxt,
|
|
&ulOutLen))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
Assert(ulOffsetToTxt + ulOutLen + 1 < m_ulMaxTokenSize + 4);
|
|
pwcsCurrency[ulOffsetToTxt + ulOutLen] = wchCurrency;
|
|
pwcsCurrency[ulOffsetToTxt + ulOutLen + 1] = L'\0';
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is a currency",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
OutputCurrency(ulOutLen+1, pwcsCurrency + ulOffsetToTxt , State, pCliticsTerm);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool CTokenizer::VerifyCommersialSign()
|
|
{
|
|
CTokenState State(m_pCurToken->m_State);
|
|
CPropFlag CommPunctTail(COMMERSIAL_SIGN_PUNCT_TAIL);
|
|
CPropFlag CommPunctHead(COMMERSIAL_SIGN_PUNCT_HEAD);
|
|
|
|
if (TEST_PROP(State.m_Properties, (COMMERSIAL_SIGN_PUNCT_TAIL | COMMERSIAL_SIGN_PUNCT_HEAD)))
|
|
{
|
|
ULONG ulCharRemoved = m_pCurToken->RemoveTailPunct(CommPunctTail, State);
|
|
ulCharRemoved += m_pCurToken->RemoveHeadPunct(CommPunctHead, State);
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
|
|
if (TEST_PROP(GET_PROP(State.m_pwcsToken[State.m_ulEnd - 1]),
|
|
PROP_COMMERSIAL_SIGN))
|
|
{
|
|
//
|
|
// the length of the token must be greater then 1 since it includes an alpha
|
|
// and the commersial sign
|
|
//
|
|
Assert((State.m_ulEnd - State.m_ulStart) > 1);
|
|
OutputCommersialSignToken(State);
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
void CTokenizer::ProcessDefault()
|
|
{
|
|
CTokenState State(m_pCurToken->m_State);
|
|
|
|
if (TEST_PROP(State.m_Properties, PROP_DEFAULT_BREAKER))
|
|
{
|
|
if (TEST_PROP(State.m_Properties, PROP_FIRST_LEVEL_BREAKER))
|
|
{
|
|
CPropFlag prop(PROP_FIRST_LEVEL_BREAKER);
|
|
|
|
BreakCompundString(State, prop);
|
|
|
|
return;
|
|
}
|
|
|
|
if (TEST_PROP(State.m_Properties, PROP_SECOND_LEVEL_BREAKER))
|
|
{
|
|
CPropFlag prop(PROP_SECOND_LEVEL_BREAKER);
|
|
|
|
BreakCompundString(State, prop);
|
|
|
|
return;
|
|
}
|
|
}
|
|
|
|
//
|
|
// this is a simple token
|
|
//
|
|
|
|
const CCliticsTerm* pCliticsTerm;
|
|
pCliticsTerm = VerifyClitics(State);
|
|
|
|
if (pCliticsTerm == &g_EmptyClitics)
|
|
{
|
|
if (TEST_PROP(State.m_Properties, PROP_NBS))
|
|
{
|
|
CPropFlag prop(PROP_NBS);
|
|
|
|
BreakCompundString(State, prop);
|
|
|
|
return;
|
|
}
|
|
|
|
CPropFlag PunctHead(SIMPLE_PUNCT_HEAD);
|
|
CPropFlag PunctTail(SIMPLE_PUNCT_TAIL);
|
|
|
|
if (TEST_PROP(State.m_Properties,
|
|
(SIMPLE_PUNCT_HEAD | SIMPLE_PUNCT_TAIL)))
|
|
{
|
|
ULONG ulCharRemoved;
|
|
ulCharRemoved= m_pCurToken->RemoveHeadPunct(PunctHead, State);
|
|
ulCharRemoved += m_pCurToken->RemoveTailPunct(PunctTail, State);
|
|
|
|
if ( TEST_PROP(State.m_Properties, PROP_UNDERSCORE) )
|
|
{
|
|
|
|
bool hasFrontUnderscore =
|
|
(State.m_ulStart > m_pCurToken->m_State.m_ulStart) &&
|
|
TEST_PROP( GET_PROP(State.m_pwcsToken[State.m_ulStart-1]),
|
|
PROP_UNDERSCORE ) &&
|
|
TEST_PROP( GET_PROP(State.m_pwcsToken[State.m_ulStart]),
|
|
PROP_ALPHA_NUMERIC );
|
|
|
|
bool hasBackUnderscore =
|
|
(State.m_ulEnd < m_pCurToken->m_State.m_ulEnd) &&
|
|
TEST_PROP(GET_PROP(State.m_pwcsToken[State.m_ulEnd]),
|
|
PROP_UNDERSCORE) &&
|
|
TEST_PROP(GET_PROP(State.m_pwcsToken[State.m_ulEnd-1]),
|
|
PROP_ALPHA_NUMERIC);
|
|
|
|
//
|
|
// Note: To change the policy to "leave ALL attached underscore
|
|
// seuences, simply change below condition to:
|
|
// if ( (hasFrontUnderscore || hasBackUnderscore) )
|
|
//
|
|
|
|
if ( (hasFrontUnderscore ^ hasBackUnderscore) )
|
|
{
|
|
ulCharRemoved -=
|
|
|
|
AddBackUnderscores(
|
|
State,
|
|
hasFrontUnderscore,
|
|
hasBackUnderscore
|
|
);
|
|
|
|
}
|
|
|
|
} // if ( TEST_PROP(State.m_Properties, PROP_UNDERSCORE) )
|
|
|
|
if (ulCharRemoved)
|
|
{
|
|
m_pCurToken->ComputeStateProperties(State);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (State.m_ulEnd == State.m_ulStart)
|
|
{
|
|
//
|
|
// case we remove all chracters in the above statement
|
|
//
|
|
return;
|
|
}
|
|
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerDecision,
|
|
("%*.*S is a simple token",
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart
|
|
));
|
|
|
|
OutputSimpleToken(State, pCliticsTerm);
|
|
}
|
|
|
|
//
|
|
// CTokenizer::AddBackUnderscores:
|
|
//
|
|
// Treat cases of a "simple" token with head and/or tail underscore
|
|
// sequence (consecutive underscores prefix or suffix); those
|
|
// do not get flipped off and remain part of the token.
|
|
// This routine is called after underscore removal, (as a result of
|
|
// Remove[Head|Tail]Punct) and adds them back in.
|
|
//
|
|
// return value: Number of underscores added back in.
|
|
//
|
|
ULONG
|
|
CTokenizer::AddBackUnderscores(
|
|
IN CTokenState& State,
|
|
IN bool hasFrontUnderscore,
|
|
IN bool hasBackUnderscore
|
|
)
|
|
{
|
|
ULONG ulCharsAdded = 0;
|
|
|
|
if ( hasFrontUnderscore )
|
|
{
|
|
// Move left over consecutive underscores
|
|
ulCharsAdded = m_pCurToken->FindLeftmostUnderscore(State);
|
|
|
|
}
|
|
|
|
if ( hasBackUnderscore )
|
|
{
|
|
|
|
// Move right over consecutive underscores
|
|
ulCharsAdded += m_pCurToken->FindRightmostUnderscore(State);
|
|
|
|
} // if ( hasFrontUnderscore )
|
|
|
|
return ulCharsAdded;
|
|
|
|
} // CTokenizer::AddBackUnderscores()
|
|
|
|
void CTokenizer::OutputUrl(CTokenState& State)
|
|
{
|
|
HRESULT hr;
|
|
|
|
ULONG ulOffsetInTxtSourceBuffer =
|
|
m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
|
|
ULONG ulCur = State.m_ulStart;
|
|
ULONG ulStart = ulCur;
|
|
ULONG ulLenInTxtSourceBuffer = 0;
|
|
ULONG ulOffsetDueToAnEscapeChar;
|
|
|
|
while (ulCur < State.m_ulEnd)
|
|
{
|
|
ulLenInTxtSourceBuffer++;
|
|
ulOffsetDueToAnEscapeChar = 0;
|
|
|
|
if ((State.m_pwcsToken[ulCur] == L'%') &&
|
|
(ulCur <= State.m_ulEnd - 2))
|
|
{
|
|
//
|
|
// replacing escape charaters with real ones.
|
|
//
|
|
if (TEST_PROP(GET_PROP(State.m_pwcsToken[ulCur+1]) , PROP_XDIGIT) &&
|
|
TEST_PROP(GET_PROP(State.m_pwcsToken[ulCur+2]) , PROP_XDIGIT))
|
|
{
|
|
short sVal;
|
|
sVal = ConvertHexCharToNumber(State.m_pwcsToken[ulCur + 1]);
|
|
sVal *= 16;
|
|
sVal += ConvertHexCharToNumber(State.m_pwcsToken[ulCur + 2]);
|
|
|
|
State.m_pwcsToken[ulCur+2] = sVal;
|
|
for (ULONG ul = ulCur -1 ; ul >= ulStart; ul--)
|
|
{
|
|
State.m_pwcsToken[ul+2] = State.m_pwcsToken[ul];
|
|
}
|
|
ulCur += 2;
|
|
ulStart+=2;
|
|
ulOffsetDueToAnEscapeChar = 2;
|
|
ulLenInTxtSourceBuffer += 2;
|
|
}
|
|
else if ((ulCur <= State.m_ulEnd - 5) &&
|
|
((State.m_pwcsToken[ulCur+1] == L'u') ||
|
|
(State.m_pwcsToken[ulCur+1] == L'U')) &&
|
|
TEST_PROP(GET_PROP(State.m_pwcsToken[ulCur+2]) , PROP_XDIGIT) &&
|
|
TEST_PROP(GET_PROP(State.m_pwcsToken[ulCur+3]) , PROP_XDIGIT) &&
|
|
TEST_PROP(GET_PROP(State.m_pwcsToken[ulCur+4]) , PROP_XDIGIT) &&
|
|
TEST_PROP(GET_PROP(State.m_pwcsToken[ulCur+5]) , PROP_XDIGIT))
|
|
{
|
|
short sVal;
|
|
sVal = ConvertHexCharToNumber(State.m_pwcsToken[ulCur + 2]);
|
|
sVal *= 0x1000;
|
|
sVal += ConvertHexCharToNumber(State.m_pwcsToken[ulCur + 3]);
|
|
sVal *= 0x100;
|
|
sVal += ConvertHexCharToNumber(State.m_pwcsToken[ulCur + 4]);
|
|
sVal *= 0x10;
|
|
sVal += ConvertHexCharToNumber(State.m_pwcsToken[ulCur + 5]);
|
|
|
|
State.m_pwcsToken[ulCur+5] = sVal;
|
|
|
|
for (ULONG ul = ulCur -1 ; ul >= ulStart; ul--)
|
|
{
|
|
State.m_pwcsToken[ul+5] = State.m_pwcsToken[ul];
|
|
}
|
|
ulCur += 5;
|
|
ulStart+=5;
|
|
ulOffsetDueToAnEscapeChar = 5;
|
|
ulLenInTxtSourceBuffer += 5;
|
|
}
|
|
}
|
|
|
|
if ( IS_BREAKER( State.m_pwcsToken[ulCur] ) )
|
|
{
|
|
if (ulCur - ulStart == 0)
|
|
{
|
|
//
|
|
// only punctuation
|
|
//
|
|
ulCur++;
|
|
ulStart = ulCur;
|
|
ulOffsetInTxtSourceBuffer += ulOffsetDueToAnEscapeChar + 1;
|
|
ulLenInTxtSourceBuffer = 0;
|
|
continue;
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
ulCur - ulStart,
|
|
&State.m_pwcsToken[ulStart],
|
|
ulLenInTxtSourceBuffer - 1 - ulOffsetDueToAnEscapeChar,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
ulStart = ulCur + 1;
|
|
ulOffsetInTxtSourceBuffer += ulLenInTxtSourceBuffer;
|
|
ulLenInTxtSourceBuffer = 0;
|
|
|
|
}
|
|
ulCur++;
|
|
|
|
}
|
|
|
|
//
|
|
// last word.
|
|
//
|
|
|
|
if (ulStart < ulCur)
|
|
{
|
|
hr = m_apWordSink->PutWord(
|
|
ulCur - ulStart,
|
|
&State.m_pwcsToken[ulStart],
|
|
ulLenInTxtSourceBuffer,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
void CTokenizer::OutputNumbers(
|
|
CTokenState& State,
|
|
ULONG ulLen,
|
|
WCHAR* pwcsNumber,
|
|
const CCliticsTerm* pCliticsTerm)
|
|
{
|
|
HRESULT hr;
|
|
//
|
|
// Input: 1.22 Output: 1.22, NN1D22
|
|
//
|
|
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
|
|
if (ulLen > m_ulMaxTokenSize)
|
|
{
|
|
hr = m_apWordSink->PutWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
return;
|
|
}
|
|
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
if (pCliticsTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart - pCliticsTerm->ulLen,
|
|
State.m_pwcsToken + State.m_ulStart + pCliticsTerm->ulLen,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
else if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart - pCliticsTerm->ulLen,
|
|
State.m_pwcsToken + State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
ulLen,
|
|
pwcsNumber,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
|
|
}
|
|
|
|
void CTokenizer::OutputParens(CTokenState& State)
|
|
{
|
|
HRESULT hr;
|
|
//
|
|
// format is xxx(s)
|
|
// Input: xxx(s) Output: xxx
|
|
//
|
|
|
|
State.m_pwcsToken[State.m_ulEnd - 3] = L'\0';
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
State.m_ulEnd - 3 - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State));
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
void CTokenizer::OutputAcronym(CTokenState& State, const CCliticsTerm* pCliticsTerm)
|
|
{
|
|
HRESULT hr;
|
|
//
|
|
// Input: I.B.M Output: I.B.M, IBM
|
|
//
|
|
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
|
|
ULONG ulAddToStart = 0;
|
|
ULONG ulDecFromEnd = 0;
|
|
|
|
if (pCliticsTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
ulAddToStart = pCliticsTerm->ulLen;
|
|
}
|
|
else if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
ulDecFromEnd = pCliticsTerm->ulLen;
|
|
|
|
}
|
|
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - ulDecFromEnd - (State.m_ulStart + ulAddToStart),
|
|
State.m_pwcsToken + State.m_ulStart + ulAddToStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
ULONG ulCur = State.m_ulStart + ulAddToStart;
|
|
ULONG ulNext = ulCur;
|
|
|
|
while (ulCur < State.m_ulEnd)
|
|
{
|
|
if (!HAS_PROP_PERIOD(GET_PROP(State.m_pwcsToken[ulCur])))
|
|
{
|
|
State.m_pwcsToken[ulNext] = State.m_pwcsToken[ulCur];
|
|
ulNext++;
|
|
ulCur++;
|
|
continue;
|
|
}
|
|
ulCur++;
|
|
}
|
|
|
|
if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
ulNext - (State.m_ulStart + ulAddToStart),
|
|
State.m_pwcsToken + State.m_ulStart + ulAddToStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
ulNext - ulDecFromEnd - (State.m_ulStart + ulAddToStart),
|
|
State.m_pwcsToken + State.m_ulStart + ulAddToStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
void CTokenizer::OutputAbbreviation(CTokenState& State)
|
|
{
|
|
HRESULT hr;
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart - 1,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
void CTokenizer::OutputSpecialAbbreviation(
|
|
CTokenState& State,
|
|
CAbbTerm* pTerm,
|
|
const CCliticsTerm* pCliticsTerm)
|
|
{
|
|
HRESULT hr;
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
|
|
WCHAR* pwcsAbb = pTerm->pwcsAbb;
|
|
ULONG ulLen = pTerm->ulAbbLen;
|
|
|
|
if (pTerm->pwcsCanonicalForm)
|
|
{
|
|
pwcsAbb = pTerm->pwcsCanonicalForm;
|
|
ulLen = pTerm->ulCanLen;
|
|
}
|
|
|
|
if (TAIL_MATCH_TRUNCATE == pCliticsTerm->ulOp)
|
|
{
|
|
WCHAR pwcs[TOKENIZER_MAXBUFFERLIMIT];
|
|
wcscpy(pwcs, pwcsAbb);
|
|
wcscpy(pwcs + ulLen, pCliticsTerm->pwcs);
|
|
|
|
hr = m_apWordSink->PutAltWord(
|
|
ulLen + pCliticsTerm->ulLen,
|
|
pwcs,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
ulLen,
|
|
pwcsAbb,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
void CTokenizer::OutputHyphenation(CTokenState& State, const CCliticsTerm* pCliticsTerm)
|
|
{
|
|
//
|
|
// Input: Data-Base Output Data Base, DataBase (only in query time)
|
|
//
|
|
HRESULT hr;
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
|
|
ULONG ulAddToStart = 0;
|
|
ULONG ulDecFromEnd = 0;
|
|
|
|
if (pCliticsTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
ulAddToStart = pCliticsTerm->ulLen;
|
|
}
|
|
else if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
ulDecFromEnd = pCliticsTerm->ulLen;
|
|
}
|
|
|
|
ULONG ulCur = State.m_ulStart + ulAddToStart;
|
|
ULONG ulStart = ulCur;
|
|
ULONG ulRelPosInTxtSrcBuff = ulOffsetInTxtSourceBuffer;
|
|
|
|
if (m_bQueryTime)
|
|
{
|
|
ULONG ulNext = ulCur;
|
|
hr = m_apWordSink->StartAltPhrase();
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
ULONG ulAdd = ulAddToStart;
|
|
while (ulCur < State.m_ulEnd)
|
|
{
|
|
if ( HAS_PROP_DASH(GET_PROP(m_pCurToken->m_State.m_pwcsToken[ulCur])))
|
|
{
|
|
hr = m_apWordSink->PutWord(
|
|
ulNext - ulStart,
|
|
&State.m_pwcsToken[ulStart],
|
|
ulNext - ulStart + ulAdd,
|
|
ulRelPosInTxtSrcBuff);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
ulRelPosInTxtSrcBuff += ulNext - ulStart + 1 + ulAdd;
|
|
ulStart = ulNext;
|
|
ulCur++;
|
|
ulAdd = 0;
|
|
continue;
|
|
}
|
|
|
|
State.m_pwcsToken[ulNext] = State.m_pwcsToken[ulCur];
|
|
ulNext++;
|
|
ulCur++;
|
|
}
|
|
|
|
Assert(ulCur > ulStart);
|
|
|
|
if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
ulNext - ulStart,
|
|
&State.m_pwcsToken[ulStart],
|
|
ulNext - ulStart,
|
|
ulRelPosInTxtSrcBuff);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
ulNext - ulStart - ulDecFromEnd,
|
|
&State.m_pwcsToken[ulStart],
|
|
ulNext - ulStart,
|
|
ulRelPosInTxtSrcBuff);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
hr = m_apWordSink->StartAltPhrase();
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
ulNext - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart - ulAddToStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
ulNext - State.m_ulStart - ulDecFromEnd - ulAddToStart,
|
|
State.m_pwcsToken + State.m_ulStart + ulAddToStart,
|
|
State.m_ulEnd - State.m_ulStart + ulAddToStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
hr = m_apWordSink->EndAltPhrase();
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
else
|
|
{
|
|
ULONG ulAdd = ulAddToStart;
|
|
|
|
while (ulCur < State.m_ulEnd)
|
|
{
|
|
if (HAS_PROP_DASH(GET_PROP(m_pCurToken->m_State.m_pwcsToken[ulCur])))
|
|
{
|
|
hr = m_apWordSink->PutWord(
|
|
ulCur - ulStart,
|
|
&State.m_pwcsToken[ulStart],
|
|
ulCur - ulStart + ulAdd,
|
|
ulRelPosInTxtSrcBuff);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
ulRelPosInTxtSrcBuff += ulCur - ulStart + 1 + ulAdd;
|
|
ulStart = ulCur + 1;
|
|
ulAdd = 0;
|
|
}
|
|
ulCur++;
|
|
}
|
|
|
|
Assert(ulCur > ulStart);
|
|
|
|
if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
ulCur - ulStart,
|
|
&State.m_pwcsToken[ulStart],
|
|
ulCur - ulStart,
|
|
ulRelPosInTxtSrcBuff);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
ulCur - ulStart - ulDecFromEnd,
|
|
&State.m_pwcsToken[ulStart],
|
|
ulCur - ulStart,
|
|
ulRelPosInTxtSrcBuff);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
}
|
|
}
|
|
|
|
void CTokenizer::OutputTime(WCHAR* pwcsTime, CTokenState& State)
|
|
{
|
|
HRESULT hr;
|
|
//
|
|
// Output: TT1353
|
|
//
|
|
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
6,
|
|
pwcsTime,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
}
|
|
|
|
void CTokenizer::OutputDate(
|
|
WCHAR* pwcsDate1,
|
|
WCHAR* pwcsDate2,
|
|
CTokenState& State)
|
|
{
|
|
HRESULT hr;
|
|
//
|
|
// Output: DD19990921
|
|
//
|
|
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
|
|
if (pwcsDate2)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
10,
|
|
pwcsDate2,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
10,
|
|
pwcsDate1,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
void CTokenizer::OutputSimpleToken(CTokenState& State, const CCliticsTerm* pTerm)
|
|
{
|
|
HRESULT hr;
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
|
|
if ((TAIL_MATCH_TRUNCATE == pTerm->ulOp) ||
|
|
(HEAD_MATCH_TRUNCATE == pTerm->ulOp))
|
|
{
|
|
if (0 == ( State.m_ulEnd - State.m_ulStart - pTerm->ulLen ))
|
|
{
|
|
return;
|
|
}
|
|
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
|
|
if (pTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutWord(
|
|
State.m_ulEnd - State.m_ulStart - pTerm->ulLen,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
else
|
|
{
|
|
Assert(pTerm->ulOp == HEAD_MATCH_TRUNCATE);
|
|
hr = m_apWordSink->PutWord(
|
|
State.m_ulEnd - State.m_ulStart - pTerm->ulLen,
|
|
&State.m_pwcsToken[State.m_ulStart + pTerm->ulLen],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State));
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
|
|
void CTokenizer::OutputCurrency(
|
|
ULONG ulLen,
|
|
WCHAR* pwcsCurrency,
|
|
CTokenState& State,
|
|
const CCliticsTerm* pTerm)
|
|
{
|
|
HRESULT hr;
|
|
//
|
|
// Output: CC12.22$
|
|
//
|
|
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
|
|
if (ulLen > m_ulMaxTokenSize)
|
|
{
|
|
hr = m_apWordSink->PutWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
return;
|
|
}
|
|
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
|
|
if (pTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart - pTerm->ulLen,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
else if (pTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart - pTerm->ulLen,
|
|
&State.m_pwcsToken[State.m_ulStart + pTerm->ulLen],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
ulLen,
|
|
pwcsCurrency,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
|
|
}
|
|
|
|
void CTokenizer::OutputCommersialSignToken(
|
|
CTokenState& State)
|
|
{
|
|
HRESULT hr;
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart - 1,
|
|
State.m_pwcsToken + State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
State.m_pwcsToken + State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
void CTokenizer::OutputMisc(
|
|
CTokenState& State,
|
|
bool bPatternContainOnlyUpperCase,
|
|
ULONG ulSuffixSize,
|
|
const CCliticsTerm* pCliticsTerm)
|
|
{
|
|
HRESULT hr;
|
|
ULONG ulOffsetInTxtSourceBuffer = m_pCurToken->CalculateStateOffsetInTxtSourceBuffer(State);
|
|
|
|
ULONG ulAddToStart = 0;
|
|
ULONG ulDecFromEnd = 0;
|
|
|
|
|
|
if (pCliticsTerm->ulOp == HEAD_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart - pCliticsTerm->ulLen,
|
|
State.m_pwcsToken + State.m_ulStart + pCliticsTerm->ulLen,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
ulAddToStart = pCliticsTerm->ulLen;
|
|
}
|
|
else if (pCliticsTerm->ulOp == TAIL_MATCH_TRUNCATE)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart - pCliticsTerm->ulLen,
|
|
State.m_pwcsToken + State.m_ulStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
ulDecFromEnd = pCliticsTerm->ulLen;
|
|
}
|
|
|
|
if (!bPatternContainOnlyUpperCase)
|
|
{
|
|
hr = m_apWordSink->PutAltWord(
|
|
State.m_ulEnd - State.m_ulStart - ulAddToStart - ulDecFromEnd - ulSuffixSize,
|
|
State.m_pwcsToken + State.m_ulStart + ulAddToStart,
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
hr = m_apWordSink->PutWord(
|
|
State.m_ulEnd - State.m_ulStart,
|
|
&State.m_pwcsToken[State.m_ulStart],
|
|
State.m_ulEnd - State.m_ulStart,
|
|
ulOffsetInTxtSourceBuffer);
|
|
if (FAILED(hr))
|
|
{
|
|
THROW_HRESULT_EXCEPTION(hr);
|
|
}
|
|
|
|
}
|
|
|
|
#define NUMBER_NO_ERROR 0
|
|
#define NUMBER_SEPERATOR_ERROR 1
|
|
#define NUMBER_ERROR 2
|
|
|
|
bool CTokenizer::CheckAndCreateNumber(
|
|
WCHAR* pwcsStr,
|
|
ULONG ulLen,
|
|
WCHAR* pwcsOut,
|
|
ULONG* pulOffsetToTxt, // the actual output does not always start at the beginning of buffer
|
|
ULONG* pulOutLen)
|
|
{
|
|
|
|
int iRet;
|
|
|
|
iRet = CheckAndCreateNumber(
|
|
pwcsStr,
|
|
ulLen,
|
|
m_apLangSupport->GetDecimalSeperator(),
|
|
m_apLangSupport->GetThousandSeperator(),
|
|
pwcsOut,
|
|
pulOffsetToTxt,
|
|
pulOutLen);
|
|
if (NUMBER_NO_ERROR == iRet)
|
|
{
|
|
return true;
|
|
}
|
|
else if (NUMBER_ERROR == iRet)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
iRet = CheckAndCreateNumber(
|
|
pwcsStr,
|
|
ulLen,
|
|
L'.', // default value
|
|
0xFFFF, // no thousand sperator
|
|
pwcsOut,
|
|
pulOffsetToTxt,
|
|
pulOutLen);
|
|
if (NUMBER_NO_ERROR == iRet)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
|
|
//
|
|
// return value:
|
|
// NUMBER_NO_ERROR - success
|
|
// NUMBER_SEPERATOR_ERROR - error due to sperators
|
|
// NUMBER_ERROR - error since it's not a number.
|
|
//
|
|
|
|
int CTokenizer::CheckAndCreateNumber(
|
|
WCHAR* pwcsStr,
|
|
ULONG ulLen,
|
|
WCHAR wchSDecimal,
|
|
WCHAR wchSThousand,
|
|
WCHAR* pwcsOut,
|
|
ULONG* pulOffsetToTxt, // the actual output does not always start at the beginning of buffer
|
|
ULONG* pulOutLen)
|
|
{
|
|
Assert(ulLen > 0);
|
|
//
|
|
// assumes that the out buffer is big enough.
|
|
// looking for the following formats: 1111 1111.2222 1,111,111.222
|
|
//
|
|
|
|
ULONG ulCur = ulLen - 1;
|
|
ULONG ulNumCharsBeforDigitSeperator = 0;
|
|
ULONG ulNextChar = ulLen - 1 + 3; // +3 is for the NN at the begging of the formated token +
|
|
// additional 0 in the begining in case .50
|
|
|
|
bool fHasFraction = false;
|
|
|
|
while ((((int)(ulCur)) >= 0) &&
|
|
HAS_PROP_NUMBER(GET_PROP(pwcsStr[ulCur])))
|
|
{
|
|
pwcsOut[ulNextChar] = pwcsStr[ulCur];
|
|
ulCur--;
|
|
ulNextChar--;
|
|
ulNumCharsBeforDigitSeperator++;
|
|
}
|
|
|
|
if (ulCur == ulLen - 1)
|
|
{
|
|
//
|
|
// did not read any digits.
|
|
//
|
|
return NUMBER_ERROR;
|
|
}
|
|
|
|
if ((((int)ulCur) >= 0) && (pwcsStr[ulCur] == wchSDecimal))
|
|
{
|
|
fHasFraction = true;
|
|
pwcsOut[ulNextChar] = L'D';
|
|
ulCur--;
|
|
ulNextChar--;
|
|
ulNumCharsBeforDigitSeperator = 0;
|
|
}
|
|
|
|
ULONG ulNumOfThousandSeperator = 0;
|
|
while (((int)ulCur) >= 0)
|
|
{
|
|
if (pwcsStr[ulCur] == wchSThousand)
|
|
{
|
|
if (3 != ulNumCharsBeforDigitSeperator)
|
|
{
|
|
return NUMBER_SEPERATOR_ERROR;
|
|
}
|
|
ulNumCharsBeforDigitSeperator = 0;
|
|
ulNumOfThousandSeperator++;
|
|
}
|
|
else if(HAS_PROP_NUMBER(GET_PROP(pwcsStr[ulCur])))
|
|
{
|
|
pwcsOut[ulNextChar] = pwcsStr[ulCur];
|
|
ulNumCharsBeforDigitSeperator++;
|
|
ulNextChar--;
|
|
}
|
|
else
|
|
{
|
|
if (TEST_PROP(
|
|
GET_PROP(pwcsStr[ulCur]), PROP_DEFAULT_BREAKER))
|
|
{
|
|
return NUMBER_SEPERATOR_ERROR;
|
|
}
|
|
|
|
return NUMBER_ERROR;
|
|
}
|
|
|
|
ulCur--;
|
|
}
|
|
|
|
*pulOutLen = ulLen;
|
|
|
|
if (L'D' == pwcsOut[ulNextChar+1])
|
|
{
|
|
Assert(ulNextChar >= 2);
|
|
//
|
|
// the number has the following format .50
|
|
//
|
|
pwcsOut[ulNextChar] = L'0';
|
|
ulNextChar--;
|
|
*pulOutLen += 1;
|
|
}
|
|
|
|
Assert(ulNextChar >= 1);
|
|
pwcsOut[ulLen + 3] = L'\0';
|
|
pwcsOut[ulNextChar] = L'N';
|
|
pwcsOut[ulNextChar - 1] = L'N';
|
|
|
|
*pulOutLen = *pulOutLen + 2 - ulNumOfThousandSeperator; // don't use += because 2 - ulNextChar + 1
|
|
*pulOffsetToTxt = ulNextChar - 1;
|
|
// can be negative and since it is ULONG we
|
|
// can get the wrong result.
|
|
if (fHasFraction)
|
|
{
|
|
while (HAS_PROP_NUMBER(GET_PROP(pwcsOut[*pulOutLen + *pulOffsetToTxt - 1])) &&
|
|
(0 == ConvertCharToDigit(pwcsOut[*pulOutLen + *pulOffsetToTxt - 1])))
|
|
{
|
|
Assert(*pulOutLen > 3);
|
|
(*pulOutLen)--;
|
|
}
|
|
|
|
if (L'D' == pwcsOut[*pulOutLen + *pulOffsetToTxt - 1])
|
|
{
|
|
(*pulOutLen)--;
|
|
}
|
|
}
|
|
return NUMBER_NO_ERROR;
|
|
}
|
|
|
|
|
|
|
|
void CTokenizer::GetValuesFromDateString(
|
|
CDateTerm* pFormat,
|
|
WCHAR* pwcsDate,
|
|
LONG* plD_M1, // we can't tell in this stage whether this is a Day or a month.
|
|
LONG* plD_M2,
|
|
LONG* plYear)
|
|
{
|
|
BYTE i;
|
|
int iBase;
|
|
|
|
*plD_M1 = 0;
|
|
for ( i = pFormat->bD_M1Len, iBase = 1; i > 0; i--, iBase *= 10)
|
|
{
|
|
*plD_M1 += ConvertCharToDigit(pwcsDate[pFormat->bD_M1Offset + i - 1]) * iBase;
|
|
}
|
|
|
|
*plD_M2 = 0;
|
|
for ( i = pFormat->bD_M2Len, iBase = 1; i > 0; i--, iBase *= 10)
|
|
{
|
|
*plD_M2 += ConvertCharToDigit(pwcsDate[pFormat->bD_M2Offset + i - 1]) * iBase;
|
|
}
|
|
|
|
*plYear = 0;
|
|
for ( i = pFormat->bYearLen, iBase = 1; i > 0; i--, iBase *= 10)
|
|
{
|
|
*plYear += ConvertCharToDigit(pwcsDate[pFormat->bYearOffset + i - 1]) * iBase;
|
|
}
|
|
|
|
}
|
|
|
|
void CTokenizer::GetValuesFromTimeString(
|
|
CTimeTerm* pFormat,
|
|
WCHAR* pwcsTime,
|
|
LONG* plHour,
|
|
LONG* plMin,
|
|
LONG* plSec,
|
|
TimeFormat* pAmPm)
|
|
{
|
|
BYTE i;
|
|
int iBase;
|
|
|
|
*plHour = 0;
|
|
for ( i = pFormat->bHourLen, iBase = 1; i > 0; i--, iBase *= 10)
|
|
{
|
|
*plHour += ConvertCharToDigit(pwcsTime[pFormat->bHourOffset + i - 1]) * iBase;
|
|
}
|
|
|
|
*plMin = 0;
|
|
for ( i = pFormat->bMinLen, iBase = 1; i > 0; i--, iBase *= 10)
|
|
{
|
|
*plMin += ConvertCharToDigit(pwcsTime[pFormat->bMinOffset + i - 1]) * iBase;
|
|
}
|
|
|
|
*plSec = 0;
|
|
for ( i = pFormat->bSecLen, iBase = 1; i > 0; i--, iBase *= 10)
|
|
{
|
|
*plSec += ConvertCharToDigit(pwcsTime[pFormat->bSecOffset + i - 1]) * iBase;
|
|
}
|
|
|
|
*pAmPm = pFormat->AmPm;
|
|
|
|
}
|
|
|
|
void CTokenizer::BreakCompundString(CTokenState& State, CPropFlag& propBreaker)
|
|
{
|
|
//
|
|
// still there are puctutaitons inside the token
|
|
// we break them up and resubmit them.
|
|
//
|
|
ULONG ulStart = State.m_ulStart;
|
|
ULONG ulCur = ulStart;
|
|
|
|
while (ulCur < State.m_ulEnd)
|
|
{
|
|
if ( TEST_PROP1(GET_PROP(State.m_pwcsToken[ulCur]), propBreaker))
|
|
{
|
|
if (ulCur - ulStart == 0)
|
|
{
|
|
//
|
|
// only punctuation
|
|
//
|
|
ulCur++;
|
|
ulStart = ulCur;
|
|
continue;
|
|
}
|
|
|
|
m_pCurToken->m_State.m_ulStart = 0;
|
|
m_pCurToken->m_State.m_ulEnd = ulCur - ulStart;
|
|
m_pCurToken->m_State.m_pwcsToken = State.m_pwcsToken + ulStart;
|
|
m_pCurToken->ComputeStateProperties(m_pCurToken->m_State);
|
|
//
|
|
// we just created a sub token need to procces it
|
|
//
|
|
|
|
ProcessTokenInternal();
|
|
ulStart = ulCur + 1;
|
|
|
|
}
|
|
ulCur++;
|
|
}
|
|
|
|
if (ulStart < ulCur)
|
|
{
|
|
//
|
|
// last sub token
|
|
//
|
|
m_pCurToken->m_State.m_ulStart = 0;
|
|
m_pCurToken->m_State.m_ulEnd = ulCur - ulStart;
|
|
m_pCurToken->m_State.m_pwcsToken = State.m_pwcsToken + ulStart;
|
|
m_pCurToken->ComputeStateProperties(m_pCurToken->m_State);
|
|
//
|
|
// we just created a sub token need to procces it
|
|
//
|
|
|
|
ProcessTokenInternal();
|
|
}
|
|
|
|
return;
|
|
|
|
}
|