424 lines
17 KiB
Plaintext
424 lines
17 KiB
Plaintext
%{
|
|
//--------------------------------------------------------------------
|
|
// Microsoft Monarch
|
|
//
|
|
// Copyright (c) Microsoft Corporation, 1997 - 1999.
|
|
//
|
|
// @doc OPTIONAL EXTRACTION CODES
|
|
//
|
|
// @module ms-sql.l |
|
|
// LEX tokenizer script
|
|
//
|
|
// @devnotes none
|
|
//
|
|
// @rev 0 | 04-Feb-97 | v-charca | Created
|
|
//
|
|
|
|
/**
|
|
** NOTE : when adding a new token (XXX) modify the following:
|
|
** 1.) Add %token _XXX to sql.y
|
|
** 2.) Add lexeme pattern to sql.l stating whether the token returns a TOKEN
|
|
** or a VALUE. If the token returns a value a node will need to be created to
|
|
** contain the value information. Therefore the VALUE macro will also need to
|
|
** specify a valid VARIANT type for the value.
|
|
**/
|
|
|
|
|
|
#include "msidxtr.h"
|
|
|
|
#ifdef DEBUG
|
|
# define YYTRACE(tknNum) LexerTrace(yytext, yyleng, tknNum);
|
|
#else
|
|
# define YYTRACE(tknNum)
|
|
#endif
|
|
|
|
|
|
#define TOKEN(tknNum) YYTRACE(tknNum) return(tknNum);
|
|
|
|
#define VALUE(tknNum) \
|
|
{ \
|
|
YYTRACE(tknNum) \
|
|
CreateTknValue(yylval, tknNum); \
|
|
return tknNum; \
|
|
}
|
|
|
|
#define STRING_VALUE(tknNum, wch, fQuote) \
|
|
{ \
|
|
YYTRACE(tknNum) \
|
|
CreateTknValue(yylval, tknNum, wch, fQuote); \
|
|
return tknNum; \
|
|
}
|
|
|
|
#define ID_VALUE(tknNum, wch) \
|
|
{ \
|
|
YYTRACE(tknNum) \
|
|
CreateTknValue(yylval, tknNum, wch); \
|
|
return _ID; \
|
|
}
|
|
|
|
/*
|
|
** Make Lex read from a block of data
|
|
** buffer is the character buffer,
|
|
** result is a variable to store the number of chars read
|
|
** ms is the size of the buffer
|
|
*/
|
|
#undef YY_INPUT
|
|
#define YY_INPUT(b, r, ms) (r = yybufferinput(b, ms))
|
|
|
|
//--------------------------------------------------------------------------------------------
|
|
// @func Makes a new copy of UNICODE string. Filters out double quotes
|
|
// @side Allocates enough bytes to hold string
|
|
// @rdesc Pointer to new UNICODE string
|
|
|
|
LPWSTR PwszDupFilter(
|
|
LPWSTR pwszOrig,
|
|
WCHAR wch )
|
|
{
|
|
LPWSTR pwszCopy = (LPWSTR)CoTaskMemAlloc( (wcslen(pwszOrig)+2)*sizeof(WCHAR) );
|
|
if ( 0 != pwszCopy )
|
|
{
|
|
LPWSTR pwsz = pwszCopy;
|
|
while ( 0 != *pwszOrig )
|
|
{
|
|
if ( *(pwszOrig+1) && *(pwszOrig+1) == *pwszOrig && wch == *pwszOrig )
|
|
pwszOrig++;
|
|
else
|
|
*pwsz++ = *pwszOrig++;
|
|
}
|
|
*pwsz = L'\0';
|
|
}
|
|
|
|
return pwszCopy;
|
|
}
|
|
|
|
//--------------------------------------------------------------------------------------------
|
|
// YYLEXER::CreateTknValue
|
|
// Creates a QUERYTREE node structure which is passed to the YACC value stack.
|
|
// This routines uses the TokenInfo map to determine which opids to create for
|
|
// the given string.
|
|
//
|
|
//
|
|
void YYLEXER::CreateTknValue(
|
|
YYSTYPE *ppct,
|
|
short tknNum,
|
|
YY_CHAR wch,
|
|
BOOL fQuote )
|
|
{
|
|
// Note that values containing variants can only be CONSTANTS or ID's
|
|
// SHOULD BE DONE BY valType
|
|
switch ( tknNum )
|
|
{
|
|
case _ID:
|
|
case _TEMPVIEW:
|
|
{
|
|
// Assume table_name for now. Might have to correct this when I
|
|
// see the context in the parser.
|
|
if ( 0 == (*ppct = PctAllocNode(DBVALUEKIND_WSTR)) )
|
|
throw(E_OUTOFMEMORY);
|
|
|
|
(*ppct)->op = DBOP_table_name;
|
|
(*ppct)->wKind = DBVALUEKIND_WSTR;
|
|
(*ppct)->value.pwszValue = CoTaskStrDup(yytext_ptr);
|
|
if( 0 == (*ppct)->value.pwszValue )
|
|
{
|
|
DeleteDBQT( *ppct );
|
|
*ppct = NULL;
|
|
throw(E_OUTOFMEMORY);
|
|
}
|
|
_wcsupr((*ppct)->value.pwszValue);
|
|
break;
|
|
}
|
|
|
|
case _DELIMITED_ID:
|
|
{
|
|
if ( 0 == (*ppct = PctAllocNode(DBVALUEKIND_WSTR)) )
|
|
throw(E_OUTOFMEMORY);
|
|
|
|
(*ppct)->op = DBOP_table_name;
|
|
(*ppct)->wKind = DBVALUEKIND_WSTR;
|
|
// Strip quotes on delimited identifier
|
|
yytext_ptr[wcslen(yytext_ptr)-1] = L'\0';
|
|
(*ppct)->value.pwszValue = PwszDupFilter(yytext_ptr+1, wch);
|
|
if( 0 == (*ppct)->value.pwszValue )
|
|
{
|
|
DeleteDBQT( *ppct );
|
|
*ppct = NULL;
|
|
throw(E_OUTOFMEMORY);
|
|
}
|
|
break;
|
|
}
|
|
|
|
case _URL:
|
|
case _STRING:
|
|
case _PREFIX_STRING:
|
|
{
|
|
// NOTE: This is really a PROPVARIANT node, but there is no DBVALUEKIND for PROPVARIANT.
|
|
if ( 0 == (*ppct = PctAllocNode(DBVALUEKIND_VARIANT, DBOP_scalar_constant)) )
|
|
throw(E_OUTOFMEMORY);
|
|
|
|
LPWSTR pwsz = yytext_ptr;
|
|
LPWSTR pwszCopy = PwszDupFilter(pwsz, wch);
|
|
|
|
if ( 0 == pwszCopy )
|
|
{
|
|
DeleteDBQT( *ppct );
|
|
*ppct = NULL;
|
|
throw(E_OUTOFMEMORY);
|
|
}
|
|
|
|
LPWSTR pwszTemp = pwszCopy;
|
|
// Strip quotes on literals or
|
|
if ( fQuote && (*pwszCopy == L'\"' || *pwszCopy == L'\'') )
|
|
{
|
|
pwszCopy++;
|
|
Assert(pwszCopy[wcslen(pwszCopy)-1] == L'\"' || pwszCopy[wcslen(pwszCopy)-1] == L'\'');
|
|
pwszCopy[wcslen(pwszCopy)-1] = L'\0';
|
|
}
|
|
|
|
((PROPVARIANT*)(*ppct)->value.pvValue)->bstrVal = SysAllocString( pwszCopy );
|
|
CoTaskMemFree( pwszTemp ); // throw away temporary before testing for out of memory
|
|
((PROPVARIANT*)(*ppct)->value.pvValue)->vt = VT_BSTR;
|
|
if( 0 == ((PROPVARIANT*)(*ppct)->value.pvValue)->bstrVal )
|
|
{
|
|
DeleteDBQT( *ppct );
|
|
*ppct = 0;
|
|
throw(E_OUTOFMEMORY);
|
|
}
|
|
}
|
|
break;
|
|
|
|
case _INTNUM:
|
|
if ( 0 == (*ppct = PctAllocNode(DBVALUEKIND_VARIANT, DBOP_scalar_constant)) )
|
|
throw(E_OUTOFMEMORY);
|
|
|
|
((PROPVARIANT*)(*ppct)->value.pvValue)->bstrVal = SysAllocString( yytext_ptr );
|
|
((PROPVARIANT*)(*ppct)->value.pvValue)->vt = VT_BSTR;
|
|
if ( 0 == ((PROPVARIANT*)(*ppct)->value.pvValue)->bstrVal )
|
|
{
|
|
DeleteDBQT( *ppct );
|
|
*ppct = 0;
|
|
throw(E_OUTOFMEMORY);
|
|
}
|
|
|
|
(*ppct)->hrError = PropVariantChangeTypeI64( (PROPVARIANT*)(*ppct)->value.pvValue );
|
|
if ( FAILED((*ppct)->hrError) )
|
|
{
|
|
HRESULT hr = (*ppct)->hrError;
|
|
DeleteDBQT( *ppct );
|
|
*ppct = 0;
|
|
throw(hr);
|
|
}
|
|
break;
|
|
|
|
case _REALNUM:
|
|
if ( 0 == (*ppct = PctAllocNode(DBVALUEKIND_VARIANT, DBOP_scalar_constant)) )
|
|
throw(E_OUTOFMEMORY);
|
|
|
|
((PROPVARIANT*)(*ppct)->value.pvValue)->bstrVal = SysAllocString( yytext_ptr );
|
|
((PROPVARIANT*)(*ppct)->value.pvValue)->vt = VT_BSTR;
|
|
if ( 0 == ((PROPVARIANT*)(*ppct)->value.pvValue)->bstrVal )
|
|
{
|
|
DeleteDBQT( *ppct );
|
|
*ppct = NULL;
|
|
throw(E_OUTOFMEMORY);
|
|
}
|
|
|
|
(*ppct)->hrError = VariantChangeTypeEx( (*ppct)->value.pvarValue, // convert in place
|
|
(*ppct)->value.pvarValue,
|
|
LOCALE_SYSTEM_DEFAULT,
|
|
0,
|
|
VT_R8 );
|
|
if ( FAILED((*ppct)->hrError) )
|
|
{
|
|
HRESULT hr = (*ppct)->hrError;
|
|
DeleteDBQT( *ppct );
|
|
*ppct = 0;
|
|
throw(hr);
|
|
}
|
|
break;
|
|
|
|
default:
|
|
Assert( !"Unkown token value" );
|
|
}
|
|
}
|
|
|
|
|
|
%}
|
|
%x contains
|
|
%x cntntsrch
|
|
%x scope0
|
|
%x scope1
|
|
%x scope2
|
|
%x view
|
|
|
|
white [ \t\n\f\r]+
|
|
id [a-zA-Z][a-zA-Z0-9_]*
|
|
simpleterm ([^ \n\t\f\r\'\(\)\[\]\&\|\~\!\,]+|\'\')*
|
|
br_id ([^\"\n]*|\"\")*
|
|
integer [-+]?[0-9]+|[-+]?0x[a-fA-F0-9]+
|
|
real [-+]?([0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?|-?\.[0-9]+([eE][-+]?[0-9]+)?
|
|
quoted_string \'([^'\n]*|\'\')*\'
|
|
string \'[^'\n]*\'
|
|
comment --[^\n]*
|
|
%%
|
|
|
|
%{
|
|
/***
|
|
*** Reserved words in every context
|
|
***/
|
|
%}
|
|
ALL { TOKEN(_ALL); }
|
|
AND { TOKEN(_AND); }
|
|
ANY { TOKEN(_ANY); }
|
|
ARRAY { TOKEN(_ARRAY); }
|
|
AS { TOKEN(_AS); }
|
|
ASC { TOKEN(_ASC); }
|
|
CAST { TOKEN(_CAST); }
|
|
CREATE { TOKEN(_CREATE); }
|
|
CONTAINS { BEGIN contains;TOKEN(_CONTAINS); }
|
|
DESC { TOKEN(_DESC); }
|
|
DROP { TOKEN(_DROP); }
|
|
FALSE { TOKEN(_FALSE); }
|
|
FREETEXT { TOKEN(_FREETEXT); }
|
|
FROM { TOKEN(_FROM); }
|
|
IS { TOKEN(_IS); }
|
|
IS{white}NOT { TOKEN(_IS_NOT); }
|
|
LIKE { TOKEN(_LIKE); }
|
|
MATCHES { TOKEN(_MATCHES); }
|
|
NOT { TOKEN(_NOT); }
|
|
NOT{white}LIKE { TOKEN(_NOT_LIKE); }
|
|
NULL { TOKEN(_NULL); }
|
|
OR { TOKEN(_OR); }
|
|
ORDER{white}BY { TOKEN(_ORDER_BY); }
|
|
PASSTHROUGH { TOKEN(_PASSTHROUGH); }
|
|
PROPERTYNAME { TOKEN(_PROPERTYNAME); }
|
|
PROPID { TOKEN(_PROPID); }
|
|
RANKMETHOD { TOKEN(_RANKMETHOD); }
|
|
SCOPE { BEGIN scope0; TOKEN(_SCOPE); }
|
|
SELECT { TOKEN(_SELECT); }
|
|
SET { TOKEN(_SET); }
|
|
SOME { TOKEN(_SOME); }
|
|
TABLE { TOKEN(_TABLE); }
|
|
TRUE { TOKEN(_TRUE); }
|
|
TYPE { TOKEN(_TYPE); }
|
|
UNION { TOKEN(_UNION); }
|
|
UNKNOWN { TOKEN(_UNKNOWN); }
|
|
VIEW { TOKEN(_VIEW); }
|
|
WHERE { TOKEN(_WHERE); }
|
|
|
|
{white} { /* empty lex rule */ }
|
|
{id} { VALUE(_ID); }
|
|
\#{id} { VALUE(_TEMPVIEW); }
|
|
\#\#{id} { VALUE(_TEMPVIEW); }
|
|
\"{br_id}\" { ID_VALUE(_DELIMITED_ID, L'"'); }
|
|
{quoted_string} { STRING_VALUE(_STRING, L'\'', TRUE);}
|
|
{integer} { VALUE(_INTNUM); }
|
|
{real} { VALUE(_REALNUM); }
|
|
{comment} { /* empty lex rule */ }
|
|
|
|
\>\= { TOKEN(_GE); }
|
|
\<\= { TOKEN(_LE); }
|
|
\<\> { TOKEN(_NE); }
|
|
\!\= { TOKEN(_NE); }
|
|
\. { TOKEN(_DOT); }
|
|
\.\. { BEGIN view; TOKEN(_DOTDOT); }
|
|
\.\.\. { BEGIN view; TOKEN(_DOTDOTDOT); }
|
|
\.\.SCOPE { BEGIN scope0; TOKEN(_DOTDOT_SCOPE);}
|
|
\.\.\.\SCOPE { BEGIN scope0; TOKEN(_DOTDOTDOT_SCOPE);}
|
|
. { YYTRACE(yytext[0]); return yytext[0]; }
|
|
|
|
%{
|
|
/***
|
|
*** A <contains predicate> has been started. The only things we should see are:
|
|
*** ( - matched by .
|
|
*** <column reference> - matched by {id} or "{br_id}"
|
|
*** , - matched by .
|
|
*** ' - matched by \'. Also switch to content search state (cntnsrch).
|
|
***/
|
|
%}
|
|
<contains>\' { BEGIN cntntsrch;YYTRACE(yytext[0]); return yytext[0];}
|
|
<contains>{id} { VALUE(_ID); }
|
|
<contains>\"{br_id}\" { ID_VALUE(_DELIMITED_ID, L'"'); }
|
|
<contains>{white} { /* empty lex rule */ }
|
|
<contains>. { YYTRACE(yytext[0]); return yytext[0];}
|
|
|
|
%{
|
|
/***
|
|
*** The only things we should see are:
|
|
*** <global view name> - matched by {id}
|
|
*** _TEMPVIEW - matched by \#{id} or \#\#{id}
|
|
***/
|
|
%}
|
|
<view>{id} { BEGIN INITIAL; VALUE(_ID); }
|
|
<view>\#{id} { BEGIN INITIAL; VALUE(_TEMPVIEW); }
|
|
<view>\#\#{id} { BEGIN INITIAL; VALUE(_TEMPVIEW); }
|
|
|
|
%{
|
|
/***
|
|
*** A <content search condition> has been started. There are several keywords we can see here.
|
|
*** We are also looking for a quoted string, a prefix string, or a simple term. We are taken
|
|
*** back to the initial state by a single quote (').
|
|
***/
|
|
%}
|
|
<cntntsrch>{white}AND{white} { unput(L' '); TOKEN(_AND); }
|
|
<cntntsrch>COERCE { TOKEN(_COERCE); }
|
|
<cntntsrch>ISABOUT { TOKEN(_ISABOUT); }
|
|
<cntntsrch>{white}NEAR { TOKEN(_NEAR); }
|
|
<cntntsrch>{white}NOT{white} { unput(L' '); TOKEN(_NOT); }
|
|
<cntntsrch>{white}OR{white} { unput(L' '); TOKEN(_OR); }
|
|
<cntntsrch>FORMSOF { TOKEN(_FORMSOF); }
|
|
<cntntsrch>WEIGHT { TOKEN(_WEIGHT); }
|
|
<cntntsrch>\"{br_id}\*\" { STRING_VALUE(_PREFIX_STRING, L'\'', TRUE);}
|
|
<cntntsrch>\"{br_id}\" { STRING_VALUE(_STRING, L'\'', TRUE);}
|
|
<cntntsrch>\' { BEGIN INITIAL; YYTRACE(yytext[0]); return yytext[0];}
|
|
<cntntsrch>{white} { /* empty lex rule */ }
|
|
<cntntsrch>{simpleterm} { STRING_VALUE(_STRING, L'\'', FALSE)}
|
|
<cntntsrch>. { YYTRACE(yytext[0]); return yytext[0];}
|
|
|
|
%{
|
|
/***
|
|
*** A <from clause> has been started. We've already seen the keyword SCOPE, so this
|
|
*** is not a FROM <view name>. We're just looking for a ( now to put us into the
|
|
*** next state (scope1).
|
|
*** ( - matched by \(. Also switch to scope1 state.
|
|
***/
|
|
%}
|
|
<scope0>\( { BEGIN scope1; YYTRACE(yytext[0]); return yytext[0];}
|
|
<scope0>{white} { /* empty lex rule */ }
|
|
<scope0>. { BEGIN scope1; YYTRACE(yytext[0]); return yytext[0];}
|
|
|
|
%{
|
|
/***
|
|
*** We're in the middle of a <from clause>. We've seen FROM SCOPE(, so now we need to recognize
|
|
*** the various scope definitions that we might see here. The two important things to recognize
|
|
*** are:
|
|
*** ( - matched by \(. Also switch to scope2 state to match parens.
|
|
*** ) - matched by \). Also switch to the initial (finished <from clause>).
|
|
***/
|
|
%}
|
|
<scope1>\"{br_id}\" { STRING_VALUE(_URL, L'"', TRUE); }
|
|
<scope1>ALL { TOKEN(_ALL); }
|
|
<scope1>DEEP{white}TRAVERSAL { TOKEN(_DEEP_TRAVERSAL); }
|
|
<scope1>EXCLUDE{white}SEARCH{white}TRAVERSAL { TOKEN(_EXCLUDE_SEARCH_TRAVERSAL);}
|
|
<scope1>OF { TOKEN(_OF); }
|
|
<scope1>SHALLOW{white}TRAVERSAL { TOKEN(_SHALLOW_TRAVERSAL); }
|
|
<scope1>{white} { /* empty lex rule */ }
|
|
<scope1>\( { BEGIN scope2; YYTRACE(yytext[0]); return yytext[0];}
|
|
<scope1>\) { BEGIN INITIAL; YYTRACE(yytext[0]); return yytext[0];}
|
|
<scope1>. { YYTRACE(yytext[0]); return yytext[0];}
|
|
|
|
%{
|
|
/***
|
|
*** We're still in the middle of a <from clause>. So far we've seen:
|
|
*** FROM SCOPE( ... (
|
|
*** We need to find a ')' to finish out the element we're working on:
|
|
*** ) - matched by \). Also switch back to scope1 state.
|
|
***/
|
|
%}
|
|
<scope2>{white} { /* empty lex rule */ }
|
|
<scope2>\"{br_id}\" { STRING_VALUE(_URL, L'"', TRUE); }
|
|
<scope2>\) { BEGIN scope1; YYTRACE(yytext[0]); return yytext[0];}
|
|
<scope2>. { YYTRACE(yytext[0]); return yytext[0];}
|
|
|
|
%%
|