1020 lines
30 KiB
C++
1020 lines
30 KiB
C++
#include "private.h"
|
|
#include "detcbase.h"
|
|
#include "codepage.h"
|
|
#include "detcjpn.h"
|
|
#include "detckrn.h"
|
|
|
|
#include "fechrcnv.h"
|
|
|
|
#include "msencode.h"
|
|
#include "lcdetect.h"
|
|
#include "cpdetect.h"
|
|
|
|
CCpMRU *g_pCpMRU = NULL;
|
|
|
|
|
|
|
|
// Get data from registry and construct cache
|
|
HRESULT CCpMRU::Init(void)
|
|
{
|
|
BOOL bRegKeyReady = TRUE;
|
|
HRESULT hr = S_OK;
|
|
HKEY hkey;
|
|
|
|
_pCpMRU = NULL;
|
|
|
|
// HKCR\\Software\\Microsoft\internet explorer\\international\\CpMRU
|
|
if (ERROR_SUCCESS != RegOpenKeyEx(HKEY_CURRENT_USER,
|
|
REGSTR_PATH_CPMRU,
|
|
0, KEY_READ|KEY_SET_VALUE, &hkey))
|
|
{
|
|
DWORD dwAction = 0;
|
|
if (ERROR_SUCCESS != RegCreateKeyEx(HKEY_CURRENT_USER,
|
|
REGSTR_PATH_CPMRU,
|
|
0, NULL, REG_OPTION_NON_VOLATILE, KEY_ALL_ACCESS, NULL, &hkey, &dwAction))
|
|
{
|
|
bRegKeyReady = FALSE;
|
|
dwCpMRUEnable = 0;
|
|
hr = E_FAIL;
|
|
}
|
|
}
|
|
|
|
if (bRegKeyReady)
|
|
{
|
|
DWORD dwType = REG_DWORD;
|
|
DWORD dwSize = sizeof(DWORD);
|
|
BOOL bUseDefault = FALSE;
|
|
|
|
if (ERROR_SUCCESS != RegQueryValueEx(hkey, REG_KEY_CPMRU_ENABLE, 0, &dwType, (LPBYTE)&dwCpMRUEnable, &dwSize))
|
|
{
|
|
dwCpMRUEnable = 1;
|
|
RegSetValueEx(hkey, REG_KEY_CPMRU_ENABLE, 0, REG_DWORD, (LPBYTE)&dwCpMRUEnable, sizeof(dwCpMRUEnable));
|
|
}
|
|
|
|
// If fail to open registry data or find unreasonable cache parameters, use default settings
|
|
if ((ERROR_SUCCESS != RegQueryValueEx(hkey, REG_KEY_CPMRU_NUM, 0, &dwType, (LPBYTE)&dwCpMRUNum, &dwSize)) ||
|
|
(ERROR_SUCCESS != RegQueryValueEx(hkey, REG_KEY_CPMRU_INIT_HITS, 0, &dwType, (LPBYTE)&dwCpMRUInitHits, &dwSize)) ||
|
|
(ERROR_SUCCESS != RegQueryValueEx(hkey, REG_KEY_CPMRU_PERCENTAGE_FACTOR, 0, &dwType, (LPBYTE)&dwCpMRUFactor, &dwSize)) ||
|
|
(dwCpMRUNum > MAX_CPMRU_NUM) || !dwCpMRUFactor || !dwCpMRUInitHits)
|
|
{
|
|
dwCpMRUNum = DEFAULT_CPMRU_NUM;
|
|
dwCpMRUInitHits = DEFAULT_CPMRU_INIT_HITS;
|
|
dwCpMRUFactor = DEFAULT_CPMRU_FACTOR;
|
|
bUseDefault = TRUE;
|
|
|
|
// Store default value in registry
|
|
RegSetValueEx(hkey, REG_KEY_CPMRU_NUM, 0, REG_DWORD, (LPBYTE)&dwCpMRUNum, sizeof(dwCpMRUNum));
|
|
RegSetValueEx(hkey, REG_KEY_CPMRU_INIT_HITS, 0, REG_DWORD, (LPBYTE)&dwCpMRUInitHits, sizeof(dwCpMRUInitHits));
|
|
RegSetValueEx(hkey, REG_KEY_CPMRU_PERCENTAGE_FACTOR, 0, REG_DWORD, (LPBYTE)&dwCpMRUFactor, sizeof(dwCpMRUFactor));
|
|
}
|
|
|
|
dwSize = sizeof(CODEPAGE_MRU)*dwCpMRUNum;
|
|
|
|
if (!dwSize || NULL == (_pCpMRU = (PCODEPAGE_MRU)LocalAlloc(LPTR, dwSize)))
|
|
{
|
|
hr = E_FAIL;
|
|
dwCpMRUEnable = 0;
|
|
}
|
|
|
|
if (_pCpMRU && !bUseDefault)
|
|
{
|
|
dwType = REG_BINARY;
|
|
|
|
if (ERROR_SUCCESS != RegQueryValueEx(hkey, REG_KEY_CPMRU, 0, &dwType, (LPBYTE)_pCpMRU, &dwSize))
|
|
{
|
|
ZeroMemory(_pCpMRU,sizeof(CODEPAGE_MRU)*dwCpMRUNum);
|
|
}
|
|
}
|
|
RegCloseKey(hkey);
|
|
}
|
|
|
|
return hr;
|
|
}
|
|
|
|
// Update registry's cache value
|
|
CCpMRU::~CCpMRU(void)
|
|
{
|
|
HKEY hkey;
|
|
|
|
if (bCpUpdated)
|
|
{
|
|
|
|
if (RegOpenKeyEx(HKEY_CURRENT_USER,
|
|
REGSTR_PATH_CPMRU,
|
|
0, KEY_READ|KEY_SET_VALUE, &hkey) == ERROR_SUCCESS)
|
|
{
|
|
DWORD dwType = REG_BINARY;
|
|
DWORD dwSize = sizeof(CODEPAGE_MRU)*dwCpMRUNum;
|
|
if (_pCpMRU)
|
|
{
|
|
RegSetValueEx(hkey, REG_KEY_CPMRU, 0, dwType, (LPBYTE)_pCpMRU, dwSize);
|
|
LocalFree(_pCpMRU);
|
|
_pCpMRU = NULL;
|
|
}
|
|
|
|
RegCloseKey(hkey);
|
|
}
|
|
bCpUpdated = FALSE;
|
|
|
|
}
|
|
}
|
|
|
|
HRESULT CCpMRU::GetCpMRU(PCODEPAGE_MRU pCpMRU, UINT *puiCpNum)
|
|
{
|
|
DWORD dwTotalHits = 0;
|
|
UINT i;
|
|
HRESULT hr = E_FAIL;
|
|
|
|
if (!(*puiCpNum))
|
|
return E_INVALIDARG;
|
|
|
|
if (!_pCpMRU)
|
|
return hr;
|
|
|
|
if (!dwCpMRUEnable || !dwCpMRUInitHits)
|
|
{
|
|
*puiCpNum = 0;
|
|
return S_FALSE;
|
|
}
|
|
|
|
ZeroMemory(pCpMRU, sizeof(CODEPAGE_MRU)*(*puiCpNum));
|
|
|
|
// Get total hits acount
|
|
for (i=0; i<dwCpMRUNum; i++)
|
|
{
|
|
if (_pCpMRU[i].dwHistoryHits)
|
|
dwTotalHits += _pCpMRU[i].dwHistoryHits;
|
|
else
|
|
break;
|
|
}
|
|
|
|
// Not enough hits count to determin the result, keep collecting
|
|
if (dwTotalHits < dwCpMRUInitHits)
|
|
{
|
|
*puiCpNum = 0;
|
|
return S_FALSE;
|
|
}
|
|
|
|
for (i=0; i<dwCpMRUNum && i<*puiCpNum; i++)
|
|
{
|
|
// Percentage is 1/MIN_CPMRU_FACTOR
|
|
if (_pCpMRU[i].dwHistoryHits*dwCpMRUFactor/dwTotalHits < 1)
|
|
break;
|
|
}
|
|
|
|
if (i != 0)
|
|
{
|
|
CopyMemory(pCpMRU, _pCpMRU, sizeof(CODEPAGE_MRU)*(i));
|
|
*puiCpNum = i;
|
|
hr = S_OK;
|
|
}
|
|
|
|
return hr;
|
|
|
|
}
|
|
|
|
// Update code page MRU
|
|
void CCpMRU::UpdateCPMRU(DWORD dwEncoding)
|
|
{
|
|
UINT i,j;
|
|
|
|
if (!_pCpMRU)
|
|
return;
|
|
|
|
if ((dwEncoding == CP_AUTO) ||
|
|
(dwEncoding == CP_JP_AUTO) ||
|
|
(dwEncoding == CP_KR_AUTO))
|
|
return;
|
|
|
|
if (!bCpUpdated)
|
|
bCpUpdated = TRUE;
|
|
|
|
|
|
// Sorted
|
|
for (i=0; i< dwCpMRUNum; i++)
|
|
{
|
|
if (!_pCpMRU[i].dwEncoding || (_pCpMRU[i].dwEncoding == dwEncoding))
|
|
break;
|
|
}
|
|
|
|
// If not found, replace the last encoding
|
|
if (i == dwCpMRUNum)
|
|
{
|
|
_pCpMRU[dwCpMRUNum-1].dwEncoding = dwEncoding;
|
|
_pCpMRU[dwCpMRUNum-1].dwHistoryHits = 1;
|
|
}
|
|
else
|
|
{
|
|
_pCpMRU[i].dwHistoryHits ++;
|
|
|
|
// If it is an already exist encoding, change order as needed
|
|
if (_pCpMRU[i].dwEncoding)
|
|
{
|
|
for (j=i; j>0; j--)
|
|
{
|
|
if (_pCpMRU[j-1].dwHistoryHits >= _pCpMRU[i].dwHistoryHits)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
if (j < i)
|
|
{
|
|
// Simple sorting
|
|
CODEPAGE_MRU tmpCPMRU = _pCpMRU[i];
|
|
|
|
MoveMemory(&_pCpMRU[j+1], &_pCpMRU[j], (i-j)*sizeof(CODEPAGE_MRU));
|
|
_pCpMRU[j].dwEncoding = tmpCPMRU.dwEncoding;
|
|
_pCpMRU[j].dwHistoryHits = tmpCPMRU.dwHistoryHits;
|
|
|
|
}
|
|
|
|
}
|
|
else
|
|
{
|
|
_pCpMRU[i].dwEncoding = dwEncoding;
|
|
}
|
|
|
|
}
|
|
|
|
// Cached too many hits?
|
|
if (_pCpMRU[0].dwHistoryHits > 0xFFFFFFF0)
|
|
{
|
|
// Find the smallest one
|
|
// This loop will always terminate
|
|
// because at worst, it will stop at i=0 (which we know
|
|
// is a huge number from the "if" above).
|
|
for (i=dwCpMRUNum-1; ; i--)
|
|
{
|
|
if (_pCpMRU[i].dwHistoryHits > 1)
|
|
break;
|
|
}
|
|
|
|
// Decrease Cache value
|
|
for (j=0; j<dwCpMRUNum && _pCpMRU[j].dwHistoryHits; j++)
|
|
{
|
|
// We still keep those one hit encodings if any
|
|
_pCpMRU[j].dwHistoryHits /= _pCpMRU[i].dwHistoryHits;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
UINT CheckEntity(LPSTR pIn, UINT nIn)
|
|
{
|
|
UINT uiRet = 0;
|
|
UINT uiSearchRange;
|
|
UINT i;
|
|
|
|
uiSearchRange = (nIn > MAX_ENTITY_LENTH)? MAX_ENTITY_LENTH:nIn;
|
|
|
|
if (*pIn == '&')
|
|
{
|
|
for(i=0; i<uiSearchRange; i++)
|
|
{
|
|
if (pIn[i] == ';')
|
|
break;
|
|
}
|
|
if (i < uiSearchRange)
|
|
{
|
|
uiSearchRange = i+1;
|
|
// NCR Entity
|
|
if (pIn[1] == '#')
|
|
{
|
|
for (i=2; i<uiSearchRange-1; i++)
|
|
if (!IS_DIGITA(pIn[i]))
|
|
{
|
|
uiSearchRange = 0;
|
|
break;
|
|
}
|
|
}
|
|
// Name Entity
|
|
else
|
|
{
|
|
for (i=1; i<uiSearchRange-1; i++)
|
|
if (!IS_CHARA(pIn[i]))
|
|
{
|
|
uiSearchRange = 0;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
uiSearchRange = 0;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
uiSearchRange = 0;
|
|
}
|
|
|
|
return uiSearchRange;
|
|
}
|
|
|
|
void RemoveHtmlTags (LPSTR pIn, UINT *pnBytes)
|
|
//
|
|
// Remove HTML tags from pIn and compress whitespace, in-place.
|
|
// On input *pnBytes is the input length; on return *pnBytes is
|
|
// set to the resulting length.
|
|
//
|
|
// Name Entity and NCR Entity strings also removed
|
|
{
|
|
UINT nIn = *pnBytes;
|
|
UINT nOut = 0;
|
|
UINT nEntity = 0;
|
|
LPSTR pOut = pIn;
|
|
BOOL fSkippedSpace = FALSE;
|
|
|
|
|
|
while ( nIn > 0 /*&& nOut + 2 < *pnBytes */) {
|
|
|
|
if (*pIn == '<' && nIn > 1/* && !IsNoise (pIn[1])*/) {
|
|
|
|
// Discard text until the end of this tag. The handling here
|
|
// is pragmatic and imprecise; what matters is detecting mostly
|
|
// contents text, not tags or comments.
|
|
pIn++;
|
|
nIn--;
|
|
|
|
LPCSTR pSkip;
|
|
DWORD nLenSkip;
|
|
|
|
if ( nIn > 1 && *pIn == '%' )
|
|
{
|
|
pSkip = "%>"; // Skip <% to %>
|
|
nLenSkip = 2;
|
|
}
|
|
else if ( nIn > 3 && *pIn == '!' && !LowAsciiStrCmpNIA(pIn, "!--", 3) )
|
|
{
|
|
pSkip = "-->"; // Skip <!-- to -->
|
|
nLenSkip = 3;
|
|
}
|
|
else if ( nIn > 5 && !LowAsciiStrCmpNIA(pIn, "style", 5) )
|
|
{
|
|
pSkip = "</style>"; // Skip <style ...> to </style>
|
|
nLenSkip = 8;
|
|
}
|
|
else if ( nIn > 6 && !LowAsciiStrCmpNIA(pIn, "script", 6) )
|
|
{
|
|
pSkip = "</script>"; // Skip <script ...> to </script>
|
|
nLenSkip = 9;
|
|
}
|
|
else if ( nIn > 3 && !LowAsciiStrCmpNIA(pIn, "xml", 3) )
|
|
{
|
|
pSkip = "</xml>";
|
|
nLenSkip = 6;
|
|
}
|
|
else
|
|
{
|
|
pSkip = ">"; // match any end tag
|
|
nLenSkip = 1;
|
|
}
|
|
|
|
// Skip up to a case-insensitive match of pSkip / nLenSkip
|
|
|
|
while ( nIn > 0 )
|
|
{
|
|
// Spin fast up to a match of the first char.
|
|
// NOTE: the first-char compare is NOT case insensitive
|
|
// because this char is known to never be alphabetic.
|
|
|
|
while ( nIn > 0 && *pIn != *pSkip )
|
|
{
|
|
pIn++;
|
|
nIn--;
|
|
}
|
|
|
|
if ( nIn > nLenSkip && !LowAsciiStrCmpNIA(pIn, pSkip, nLenSkip) )
|
|
{
|
|
pIn += nLenSkip;
|
|
nIn -= nLenSkip;
|
|
fSkippedSpace = TRUE;
|
|
|
|
break;
|
|
}
|
|
|
|
if ( nIn > 0)
|
|
{
|
|
pIn++;
|
|
nIn--;
|
|
}
|
|
}
|
|
|
|
// *pIn is either one past '>' or at end of input
|
|
|
|
}
|
|
else
|
|
if (IsNoise (*pIn) || (nEntity = CheckEntity(pIn, nIn)))
|
|
{
|
|
|
|
// Collapse whitespace -- remember it but don't copy it now
|
|
fSkippedSpace = TRUE;
|
|
if (nEntity)
|
|
{
|
|
pIn+=nEntity;
|
|
nIn-=nEntity;
|
|
nEntity = 0;
|
|
}
|
|
else
|
|
{
|
|
while (nIn > 0 && IsNoise (*pIn))
|
|
pIn++, nIn--;
|
|
}
|
|
}
|
|
// *pIn is non-ws char
|
|
else
|
|
{
|
|
// Pass through all other characters
|
|
// Compress all previous noise characters to a white space
|
|
if (fSkippedSpace)
|
|
{
|
|
*pOut++ = ' ';
|
|
nOut++;
|
|
fSkippedSpace = FALSE;
|
|
}
|
|
|
|
*pOut++ = *pIn++;
|
|
nIn--;
|
|
nOut++;
|
|
}
|
|
}
|
|
|
|
*pnBytes = nOut;
|
|
}
|
|
|
|
static unsigned char szKoi8ru[] = {0xA4, 0xA6, 0xA7, 0xB4, 0xB6, 0xB7, 0xAD, 0xAE, 0xBD, 0xBE};
|
|
static unsigned char sz28592[] = {0xA1, 0xA6, /*0xAB,*/ 0xAC, 0xB1, 0xB5, 0xB6, 0xB9, /*0xBB, 0xE1*/}; // Need to fine tune this data
|
|
|
|
const CPPATCH CpData[] =
|
|
{
|
|
{CP_KOI8R, CP_KOI8RU, ARRAYSIZE(szKoi8ru), szKoi8ru},
|
|
{CP_1250, CP_ISO_8859_2, ARRAYSIZE(sz28592), sz28592},
|
|
};
|
|
|
|
|
|
// Distinguish similar western encodings
|
|
UINT PatchCodePage(UINT uiEncoding, unsigned char *pStr, int nSize)
|
|
{
|
|
int i, l,m, n, iPatch=0;
|
|
|
|
while (iPatch < ARRAYSIZE(CpData))
|
|
{
|
|
if (uiEncoding == CpData[iPatch].srcEncoding)
|
|
{
|
|
for (i=0; i<nSize; i++)
|
|
{
|
|
if (*pStr > HIGHEST_ASCII)
|
|
{
|
|
l = 0;
|
|
m = CpData[iPatch].nSize-1;
|
|
n = m / 2;
|
|
while (l <= m)
|
|
{
|
|
if (*pStr == CpData[iPatch].pszUch[n])
|
|
return CpData[iPatch].destEncoding;
|
|
else
|
|
{
|
|
if (*pStr < CpData[iPatch].pszUch[n])
|
|
{
|
|
m = n-1;
|
|
}
|
|
else
|
|
{
|
|
l = n+1;
|
|
}
|
|
n = (l+m)/2;
|
|
}
|
|
}
|
|
}
|
|
pStr++;
|
|
}
|
|
}
|
|
iPatch++;
|
|
}
|
|
|
|
return uiEncoding;
|
|
}
|
|
|
|
|
|
|
|
#if 0
|
|
|
|
const unsigned char szKOIRU[] = {0xA4, 0xA6, 0xA7, 0xB4, 0xB6, 0xB7, 0xAD, 0xAE, 0xBD, 0xBE};
|
|
|
|
BOOL _IsKOI8RU(unsigned char *pStr, int nSize)
|
|
{
|
|
int i,j;
|
|
BOOL bRet = FALSE;
|
|
|
|
// Skip parameter check since this is internal
|
|
for (i=0; i<nSize; i++)
|
|
{
|
|
if (*pStr >= szKOIRU[0] && *pStr <= szKOIRU[ARRAYSIZE(szKOIRU)-1])
|
|
{
|
|
for (j=0; j<ARRAYSIZE(szKOIRU); j++)
|
|
{
|
|
if (*pStr == szKOIRU[j])
|
|
{
|
|
bRet = TRUE;
|
|
break;
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
if (bRet)
|
|
break;
|
|
|
|
pStr++;
|
|
}
|
|
|
|
return bRet;
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
HRESULT WINAPI _DetectInputCodepage(DWORD dwFlag, DWORD uiPrefWinCodepage, CHAR *pSrcStr, INT *pcSrcSize, DetectEncodingInfo *lpEncoding, INT *pnScores)
|
|
{
|
|
|
|
HRESULT hr = S_OK;
|
|
IStream *pstmTmp = NULL;
|
|
BOOL bGuess = FALSE;
|
|
BOOL bLCDetectSucceed = FALSE;
|
|
int nBufSize = *pnScores;
|
|
CHAR *_pSrcStr = pSrcStr;
|
|
UINT nSrcSize;
|
|
int i;
|
|
BOOL bMayBeAscii = FALSE;
|
|
|
|
// Check parameters
|
|
if (!pSrcStr || !(*pcSrcSize) || !lpEncoding || *pnScores == 0)
|
|
return E_INVALIDARG;
|
|
|
|
nSrcSize = *pcSrcSize;
|
|
|
|
// Zero out return buffer
|
|
ZeroMemory(lpEncoding, sizeof(DetectEncodingInfo)*(*pnScores));
|
|
|
|
// Simple Unicode detection
|
|
if (nSrcSize >= sizeof(WCHAR))
|
|
{
|
|
UINT uiCp = 0;
|
|
|
|
if (*((WCHAR *)pSrcStr) == 0xFEFF) // Unicode
|
|
uiCp = CP_UCS_2;
|
|
else if (*((WCHAR *)pSrcStr) == 0xFFFE) // Uncode Big Endian
|
|
uiCp = CP_UCS_2_BE;
|
|
|
|
if (uiCp)
|
|
{
|
|
*pnScores = 1;
|
|
lpEncoding[0].nCodePage = uiCp;
|
|
lpEncoding[0].nConfidence = 100;
|
|
lpEncoding[0].nDocPercent = 100;
|
|
lpEncoding[0].nLangID = -1;
|
|
return S_OK;
|
|
}
|
|
}
|
|
|
|
// HTML: take off HTML 'decoration'
|
|
if (dwFlag & MLDETECTCP_HTML)
|
|
{
|
|
// Dup buffer for HTML parser
|
|
if (NULL == (_pSrcStr = (char *)LocalAlloc(LPTR, nSrcSize)))
|
|
return E_OUTOFMEMORY;
|
|
CopyMemory(_pSrcStr, pSrcStr, nSrcSize);
|
|
RemoveHtmlTags (_pSrcStr, &nSrcSize);
|
|
}
|
|
|
|
// if blank page/file...
|
|
if (!nSrcSize)
|
|
return E_FAIL;
|
|
|
|
if (nSrcSize >= MIN_TEXT_SIZE)
|
|
{
|
|
// Initialize LCDetect
|
|
if (NULL == g_pLCDetect)
|
|
{
|
|
EnterCriticalSection(&g_cs);
|
|
if (NULL == g_pLCDetect)
|
|
{
|
|
LCDetect *pLC = new LCDetect ((HMODULE)g_hInst);
|
|
if (pLC)
|
|
{
|
|
if (pLC->LoadState() == NO_ERROR)
|
|
g_pLCDetect = pLC;
|
|
else
|
|
{
|
|
delete pLC;
|
|
}
|
|
}
|
|
}
|
|
LeaveCriticalSection(&g_cs);
|
|
}
|
|
|
|
if (g_pLCDetect)
|
|
{
|
|
LCD_Detect(_pSrcStr, nSrcSize, (PLCDScore)lpEncoding, pnScores, NULL);
|
|
if (*pnScores)
|
|
{
|
|
hr = S_OK;
|
|
bLCDetectSucceed = TRUE;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!bLCDetectSucceed)
|
|
{
|
|
*pnScores = 0;
|
|
hr = E_FAIL;
|
|
}
|
|
|
|
unsigned int uiCodepage = 0;
|
|
LARGE_INTEGER li = {0,0};
|
|
ULARGE_INTEGER uli = {0,0};
|
|
|
|
|
|
if (S_OK == CreateStreamOnHGlobal(NULL, TRUE, &pstmTmp))
|
|
{
|
|
ULONG cb = (ULONG) nSrcSize ;
|
|
if (S_OK == pstmTmp->Write(_pSrcStr,cb,&cb))
|
|
{
|
|
uli.LowPart = cb ;
|
|
if (S_OK != pstmTmp->SetSize(uli))
|
|
{
|
|
hr = E_OUTOFMEMORY;
|
|
goto DETECT_DONE;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
goto DETECT_DONE;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
hr = E_OUTOFMEMORY;
|
|
goto DETECT_DONE;
|
|
}
|
|
|
|
pstmTmp->Seek(li,STREAM_SEEK_SET, NULL);
|
|
|
|
switch (CceDetectInputCode(pstmTmp, grfDetectResolveAmbiguity|grfDetectUseCharMapping|grfDetectIgnoreEof, (EFam) 0, 0, &uiCodepage, &bGuess))
|
|
{
|
|
case cceSuccess:
|
|
if (*pnScores)
|
|
{
|
|
// LCDETECT never detects wrong on Arabic and Russian, don't consider it as DBCS in this case
|
|
// because MSEncode might misdetect Arabic and Russian as Japanese
|
|
// Same goes for Korean JOHAB, MSENCODE doesn't support it at all
|
|
if (((lpEncoding[0].nLangID == LANG_ARABIC )|| (lpEncoding[0].nLangID == LANG_RUSSIAN) || (lpEncoding[0].nCodePage == CP_KOR_JOHAB)) &&
|
|
(lpEncoding[0].nConfidence >= MIN_ACCEPTABLE_CONFIDENCE)
|
|
&& (lpEncoding[0].nDocPercent >= MIN_DOCPERCENT) && !bGuess)
|
|
bGuess = TRUE;
|
|
|
|
for (i=0;i<*pnScores;i++)
|
|
{
|
|
if (lpEncoding[i].nCodePage == uiCodepage)
|
|
{
|
|
if ((i != 0) && !bGuess)
|
|
{
|
|
DetectEncodingInfo TmpEncoding;
|
|
// Re-arrange lanugage list for MSEncode result
|
|
MoveMemory(&TmpEncoding, &lpEncoding[0], sizeof(DetectEncodingInfo));
|
|
MoveMemory(&lpEncoding[0], &lpEncoding[i], sizeof(DetectEncodingInfo));
|
|
MoveMemory(&lpEncoding[i], &TmpEncoding, sizeof(DetectEncodingInfo));
|
|
}
|
|
// Boost confidence for double hits
|
|
lpEncoding[0].nDocPercent = 100;
|
|
if (lpEncoding[0].nConfidence < 100)
|
|
lpEncoding[0].nConfidence = 100;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (i == *pnScores)
|
|
{
|
|
if (bGuess)
|
|
{
|
|
if (nBufSize > *pnScores)
|
|
{
|
|
lpEncoding[*pnScores].nCodePage = uiCodepage;
|
|
lpEncoding[*pnScores].nConfidence = MIN_CONFIDENCE;
|
|
lpEncoding[*pnScores].nDocPercent = MIN_DOCPERCENT;
|
|
lpEncoding[*pnScores].nLangID = -1;
|
|
(*pnScores)++;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (nBufSize > *pnScores)
|
|
{
|
|
MoveMemory(lpEncoding+1, lpEncoding, sizeof(DetectEncodingInfo) * (*pnScores));
|
|
(*pnScores)++;
|
|
}
|
|
else
|
|
{
|
|
MoveMemory(lpEncoding+1, lpEncoding, sizeof(DetectEncodingInfo) * (*pnScores-1));
|
|
}
|
|
lpEncoding[0].nCodePage = uiCodepage;
|
|
lpEncoding[0].nConfidence = 100;
|
|
lpEncoding[0].nDocPercent = MIN_DOCPERCENT;
|
|
lpEncoding[0].nLangID = -1;
|
|
}
|
|
}
|
|
|
|
}
|
|
else
|
|
{
|
|
lpEncoding[0].nCodePage = uiCodepage;
|
|
if (bGuess)
|
|
lpEncoding[0].nConfidence = MIN_CONFIDENCE;
|
|
else
|
|
lpEncoding[0].nConfidence = 100;
|
|
lpEncoding[0].nDocPercent = MIN_DOCPERCENT;
|
|
lpEncoding[0].nLangID = -1;
|
|
(*pnScores)++;
|
|
}
|
|
|
|
//hr = (g_pLCDetect || (nSrcSize < MIN_TEXT_SIZE)) ? S_OK : S_FALSE;
|
|
hr = (!g_pLCDetect || (bGuess && !bLCDetectSucceed )) ? S_FALSE : S_OK;
|
|
break;
|
|
|
|
// Currently MSEncode doesn't provide any useful information in 'cceAmbiguousInput' case.
|
|
// We may update our code here if Office team enhance MSEncode for ambiguous input later.
|
|
case cceAmbiguousInput:
|
|
break;
|
|
|
|
case cceMayBeAscii:
|
|
bMayBeAscii = TRUE;
|
|
if (!(*pnScores))
|
|
{
|
|
lpEncoding[0].nCodePage = uiCodepage;
|
|
lpEncoding[0].nConfidence = MIN_CONFIDENCE;
|
|
lpEncoding[0].nDocPercent = -1;
|
|
lpEncoding[0].nLangID = -1;
|
|
(*pnScores)++;
|
|
}
|
|
else
|
|
{
|
|
for (i=0;i<*pnScores;i++)
|
|
{
|
|
if (lpEncoding[i].nCodePage == uiCodepage)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (i == *pnScores)
|
|
{
|
|
if(nBufSize > *pnScores) // Append MSEncode result to the language list
|
|
{
|
|
lpEncoding[i].nCodePage = uiCodepage;
|
|
lpEncoding[i].nConfidence = -1;
|
|
lpEncoding[i].nDocPercent = -1;
|
|
lpEncoding[i].nLangID = -1;
|
|
(*pnScores)++;
|
|
}
|
|
}
|
|
}
|
|
hr = bLCDetectSucceed ? S_OK : S_FALSE;
|
|
break;
|
|
|
|
// MSEncode failed
|
|
default:
|
|
break;
|
|
}
|
|
|
|
|
|
|
|
|
|
for (i=0; i<*pnScores; i++)
|
|
{
|
|
switch (lpEncoding[i].nCodePage) {
|
|
|
|
case 850:
|
|
if ((*pnScores>1) && (lpEncoding[1].nConfidence >= MIN_CONFIDENCE))
|
|
{
|
|
// Remove 850 from detection result if there is other detection results
|
|
(*pnScores)--;
|
|
if (i < *pnScores)
|
|
MoveMemory(&lpEncoding[i], &lpEncoding[i+1], (*pnScores-i)*sizeof(DetectEncodingInfo));
|
|
ZeroMemory(&lpEncoding[*pnScores], sizeof(DetectEncodingInfo));
|
|
}
|
|
else
|
|
{
|
|
// Replace it with 1252 if it is the only result we get
|
|
lpEncoding[0].nCodePage = CP_1252;
|
|
lpEncoding[0].nConfidence =
|
|
lpEncoding[0].nDocPercent = 100;
|
|
lpEncoding[0].nLangID = LANG_ENGLISH;
|
|
}
|
|
break;
|
|
|
|
case CP_1250:
|
|
case CP_KOI8R:
|
|
lpEncoding[i].nCodePage = PatchCodePage(lpEncoding[i].nCodePage, (unsigned char *)_pSrcStr, nSrcSize);
|
|
break;
|
|
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
// If not a high confidence CP_1254 (Windows Turkish),
|
|
// we'll check if there're better detection results, and swap results if needed
|
|
if ((lpEncoding[0].nCodePage == CP_1254) &&
|
|
(*pnScores>1) &&
|
|
((lpEncoding[0].nDocPercent < 90) || (lpEncoding[1].nCodePage == CP_CHN_GB) ||
|
|
(lpEncoding[1].nCodePage == CP_TWN) || (lpEncoding[1].nCodePage == CP_JPN_SJ) || (lpEncoding[1].nCodePage == CP_KOR_5601)))
|
|
{
|
|
MoveMemory(&lpEncoding[0], &lpEncoding[1], sizeof(DetectEncodingInfo)*(*pnScores-1));
|
|
lpEncoding[*pnScores-1].nCodePage = CP_1254;
|
|
lpEncoding[*pnScores-1].nLangID = LANG_TURKISH;
|
|
}
|
|
|
|
// 852 and 1258 text only have one sure detection result
|
|
if (((lpEncoding[0].nCodePage == CP_852) || (lpEncoding[0].nCodePage == CP_1258)) &&
|
|
(*pnScores>1) &&
|
|
(lpEncoding[1].nConfidence >= MIN_CONFIDENCE))
|
|
{
|
|
DetectEncodingInfo tmpDetect = {0};
|
|
MoveMemory(&tmpDetect, &lpEncoding[0], sizeof(DetectEncodingInfo));
|
|
MoveMemory(&lpEncoding[0], &lpEncoding[1], sizeof(DetectEncodingInfo));
|
|
MoveMemory(&lpEncoding[1], &tmpDetect, sizeof(DetectEncodingInfo));
|
|
}
|
|
|
|
// Considering guessed value from MSENCODE is pretty accurate, we don't change S_OK to S_FALSE
|
|
#if 0
|
|
if ((S_OK == hr) && !bLCDetectSucceed && bGuess)
|
|
{
|
|
hr = S_FALSE;
|
|
}
|
|
#endif
|
|
|
|
if (uiPrefWinCodepage && *pnScores)
|
|
{
|
|
if (uiPrefWinCodepage == CP_AUTO && g_pCpMRU && !IS_ENCODED_ENCODING(lpEncoding[0].nCodePage))
|
|
{
|
|
UINT uiCpNum = CP_AUTO_MRU_NUM;
|
|
CODEPAGE_MRU CpMRU[CP_AUTO_MRU_NUM];
|
|
|
|
if (S_OK == g_pCpMRU->GetCpMRU(CpMRU, &uiCpNum))
|
|
{
|
|
for (i = 0; i<*pnScores; i++)
|
|
{
|
|
for (UINT j = 0; j < uiCpNum; j++)
|
|
{
|
|
if (lpEncoding[i].nCodePage == CpMRU[j].dwEncoding)
|
|
{
|
|
uiPrefWinCodepage = CpMRU[j].dwEncoding;
|
|
break;
|
|
}
|
|
}
|
|
if (uiPrefWinCodepage != CP_AUTO)
|
|
break;
|
|
}
|
|
|
|
// If detection result is not in MRU
|
|
if (uiPrefWinCodepage == CP_AUTO)
|
|
{
|
|
// Don't take Unicode as perferred encoding if it is not in detection results for following reasons
|
|
// 1. Unicode is usually tagged with charset or Unicode BOM
|
|
// 2. Currently, we don't support Unicode detection in all detection engines
|
|
if (CpMRU[0].dwEncoding != CP_UCS_2 && CpMRU[0].dwEncoding != CP_UCS_2_BE)
|
|
uiPrefWinCodepage = CpMRU[0].dwEncoding;
|
|
}
|
|
}
|
|
}
|
|
|
|
// End preferred CP check if we can't get a valid one
|
|
if (uiPrefWinCodepage == CP_AUTO)
|
|
goto PREFERCPCHECK_DONE;
|
|
|
|
for (i = 1; i<*pnScores; i++)
|
|
{
|
|
if (uiPrefWinCodepage == lpEncoding[i].nCodePage)
|
|
{
|
|
DetectEncodingInfo TmpEncoding;
|
|
// Re-arrange lanugage list for prefered codepage
|
|
TmpEncoding = lpEncoding[i];
|
|
MoveMemory(&lpEncoding[1], &lpEncoding[0], sizeof(DetectEncodingInfo)*i);
|
|
lpEncoding[0] = TmpEncoding;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if ((uiPrefWinCodepage != lpEncoding[0].nCodePage) &&
|
|
((bMayBeAscii && (lpEncoding[0].nConfidence <= MIN_CONFIDENCE)) ||
|
|
(hr != S_OK && nSrcSize >= MIN_TEXT_SIZE) ||
|
|
(nSrcSize < MIN_TEXT_SIZE && !IS_ENCODED_ENCODING(lpEncoding[0].nCodePage))))
|
|
{
|
|
lpEncoding[0].nCodePage = uiPrefWinCodepage;
|
|
lpEncoding[0].nConfidence = -1;
|
|
lpEncoding[0].nDocPercent = -1;
|
|
lpEncoding[0].nLangID = -1;
|
|
*pnScores = 1;
|
|
}
|
|
}
|
|
|
|
PREFERCPCHECK_DONE:
|
|
|
|
// Assume LCDETECT won't misdetect 1252 for files over MIN_TEXT_SIZE
|
|
// and MSENCODE can handle encoded text even they're below MIN_TEXT_SIZE
|
|
if (((nSrcSize < MIN_TEXT_SIZE) && (bMayBeAscii || E_FAIL == hr)) ||
|
|
(lpEncoding[0].nCodePage == CP_1252) ||
|
|
(lpEncoding[0].nCodePage == CP_UTF_8))
|
|
{
|
|
UINT j;
|
|
for (j=0; j < nSrcSize; j++)
|
|
if (*((LPBYTE)(_pSrcStr+j)) > HIGHEST_ASCII)
|
|
break;
|
|
if (j == nSrcSize)
|
|
{
|
|
if (lpEncoding[0].nCodePage == CP_1252)
|
|
{
|
|
lpEncoding[0].nCodePage = CP_20127;
|
|
}
|
|
else
|
|
{
|
|
*pnScores = 1;
|
|
lpEncoding[0].nCodePage = CP_20127;
|
|
lpEncoding[0].nConfidence =
|
|
lpEncoding[0].nDocPercent = 100;
|
|
lpEncoding[0].nLangID = LANG_ENGLISH;
|
|
hr = S_OK;
|
|
}
|
|
}
|
|
}
|
|
|
|
// UTF-8 doesn't really have distinctive signatures,
|
|
// if text amout is small, we won't return low confidence UTF-8 detection result.
|
|
if (hr == S_FALSE && IS_ENCODED_ENCODING(lpEncoding[0].nCodePage) &&
|
|
!((nSrcSize < MIN_TEXT_SIZE) && (lpEncoding[0].nCodePage == CP_UTF_8)))
|
|
hr = S_OK;
|
|
|
|
DETECT_DONE:
|
|
|
|
if ((dwFlag & MLDETECTCP_HTML) && _pSrcStr)
|
|
LocalFree(_pSrcStr);
|
|
|
|
if (pstmTmp)
|
|
{
|
|
pstmTmp->Release();
|
|
}
|
|
|
|
return hr ;
|
|
}
|
|
|
|
HRESULT WINAPI _DetectCodepageInIStream(DWORD dwFlag, DWORD uiPrefWinCodepage, IStream *pstmIn, DetectEncodingInfo *lpEncoding, INT *pnScores)
|
|
{
|
|
|
|
HRESULT hr= S_OK, hrWarnings=S_OK;
|
|
LARGE_INTEGER libOrigin = { 0, 0 };
|
|
ULARGE_INTEGER ulPos = {0, 0};
|
|
LPSTR lpstrIn = NULL ;
|
|
ULONG nlSrcSize ;
|
|
INT nSrcUsed ;
|
|
|
|
if (!pstmIn)
|
|
return E_INVALIDARG ;
|
|
|
|
// get size
|
|
hr = pstmIn->Seek(libOrigin, STREAM_SEEK_END,&ulPos);
|
|
if (S_OK != hr)
|
|
hrWarnings = hr;
|
|
|
|
if ( ulPos.LowPart == 0 && ulPos.HighPart == 0 )
|
|
return E_INVALIDARG ;
|
|
|
|
nlSrcSize = ulPos.LowPart ;
|
|
|
|
// allocate a temp input buffer
|
|
if ( (lpstrIn = (LPSTR) LocalAlloc(LPTR, nlSrcSize )) == NULL )
|
|
{
|
|
hrWarnings = E_OUTOFMEMORY ;
|
|
goto exit;
|
|
}
|
|
|
|
// reset the pointer
|
|
hr = pstmIn->Seek(libOrigin, STREAM_SEEK_SET, NULL);
|
|
if (S_OK != hr)
|
|
hrWarnings = hr;
|
|
|
|
hr = pstmIn->Read(lpstrIn, nlSrcSize, &nlSrcSize);
|
|
if (S_OK != hr)
|
|
hrWarnings = hr;
|
|
|
|
nSrcUsed = (INT) nlSrcSize ;
|
|
hr = _DetectInputCodepage(dwFlag, uiPrefWinCodepage, lpstrIn, &nSrcUsed, lpEncoding, pnScores);
|
|
|
|
exit :
|
|
if (lpstrIn)
|
|
{
|
|
LocalFree(lpstrIn);
|
|
}
|
|
|
|
return (hr == S_OK) ? hrWarnings : hr;
|
|
}
|