1778 lines
58 KiB
C
1778 lines
58 KiB
C
/*************************************************************************
|
|
* *
|
|
* PERMIND2.C *
|
|
* *
|
|
* Copyright (C) Microsoft Corporation 1990-1994 *
|
|
* All Rights reserved. *
|
|
* *
|
|
**************************************************************************
|
|
* *
|
|
* Module Intent *
|
|
* This is the final stage of the index building process. This module *
|
|
* converts the input data into a permanent B-Tree file. *
|
|
* *
|
|
* Stem node structure: *
|
|
* CbLeft |* Word | PointerToNode *| Slack *
|
|
* *
|
|
* Leaf node structure: *
|
|
* NxtBlkPtr|CbLeft|*Word|FieldId|TopicCnt|PointerToNode|DataSize*|Slack *
|
|
* *
|
|
* Data node structure: *
|
|
* |* Topic | OccBlkCnt |* OccBlk *| *| Slack *
|
|
* *
|
|
* Fields between |* *| repeat based on count values *
|
|
* *
|
|
**************************************************************************
|
|
* *
|
|
* Current Owner: BinhN *
|
|
* *
|
|
**************************************************************************/
|
|
|
|
#include <mvopsys.h>
|
|
#include <mem.h>
|
|
#include <memory.h>
|
|
#include <math.h>
|
|
#include <orkin.h>
|
|
#include <mvsearch.h>
|
|
#include "common.h"
|
|
#include "index.h"
|
|
|
|
#ifdef _DEBUG
|
|
static BYTE NEAR s_aszModule[] = __FILE__; /* Used by error return functions.*/
|
|
#endif
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* PRIVATE PUBLIC FUNCTIONS
|
|
*
|
|
* All of them should be declared far, unless we know they belong to
|
|
* the same segment. They should be included in some include file
|
|
*
|
|
*************************************************************************/
|
|
|
|
PUBLIC HRESULT FAR PASCAL BuildBTree (HFPB, _LPIPB, LPB, HFPB, LPSTR);
|
|
PUBLIC PNODEINFO FAR PASCAL AllocBTreeNode (_LPIPB);
|
|
PUBLIC VOID PASCAL FAR FreeBTreeNode (PNODEINFO pNode);
|
|
PUBLIC int FAR PASCAL PrefixCompressWord (LPB, LPB, LPB, int);
|
|
PUBLIC HRESULT FAR PASCAL FWriteBits(PFILEDATA, DWORD, BYTE);
|
|
PUBLIC DWORD FAR PASCAL WriteDataNode (_LPIPB, DWORD, PHRESULT);
|
|
|
|
/*************************************************************************
|
|
*
|
|
* PRIVATE PRIVATE FUNCTIONS
|
|
*
|
|
*************************************************************************/
|
|
|
|
PRIVATE HRESULT NEAR PASCAL AddRecordToLeaf (_LPIPB);
|
|
PRIVATE HRESULT NEAR PASCAL AddRecordToStem (_LPIPB, LPB);
|
|
PRIVATE int NEAR PASCAL CompressDword (PFILEDATA, DWORD);
|
|
PRIVATE HRESULT NEAR PASCAL WriteStemNode (_LPIPB, PNODEINFO);
|
|
PRIVATE HRESULT NEAR PASCAL WriteLeafNode (_LPIPB);
|
|
PRIVATE HRESULT NEAR PASCAL FlushAllNodes (_LPIPB);
|
|
// Compression functions
|
|
// PRIVATE HRESULT NEAR PASCAL FAddDword (PFILEDATA, DWORD, CKEY);
|
|
PRIVATE HRESULT NEAR PASCAL FWriteBool(PFILEDATA, BOOL);
|
|
|
|
|
|
// This table is used to avoid the calculation "(1L << v) - 1". Instead
|
|
// you say "argdwBits[v]", which should be faster. The table is useful
|
|
// other places, too.
|
|
DWORD argdwBits[] =
|
|
{
|
|
0x00000000, 0x00000001, 0x00000003, 0x00000007,
|
|
0x0000000F, 0x0000001F, 0x0000003F, 0x0000007F,
|
|
0x000000FF, 0x000001FF, 0x000003FF, 0x000007FF,
|
|
0x00000FFF, 0x00001FFF, 0x00003FFF, 0x00007FFF,
|
|
0x0000FFFF, 0x0001FFFF, 0x0003FFFF, 0x0007FFFF,
|
|
0x000FFFFF, 0x001FFFFF, 0x003FFFFF, 0x007FFFFF,
|
|
0x00FFFFFF, 0x01FFFFFF, 0x03FFFFFF, 0x07FFFFFF,
|
|
0x0FFFFFFF, 0x1FFFFFFF, 0x3FFFFFFF, 0x7FFFFFFF,
|
|
0xFFFFFFFF,
|
|
};
|
|
|
|
PRIVATE HRESULT PASCAL NEAR WriteBitStreamDWord (PFILEDATA, DWORD, int);
|
|
PRIVATE HRESULT PASCAL NEAR WriteFixedDWord (PFILEDATA, DWORD, int);
|
|
PRIVATE HRESULT PASCAL NEAR WriteBellDWord (PFILEDATA, DWORD, int);
|
|
|
|
FENCODE EncodeTable[] =
|
|
{
|
|
WriteBitStreamDWord,
|
|
WriteFixedDWord,
|
|
WriteBellDWord,
|
|
NULL,
|
|
};
|
|
|
|
|
|
#define FAddDword(p,dw,key) (EncodeTable[(key).cschScheme]((p), (dw), (key).ucCenter))
|
|
#define SAFE_SLACK 256
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func HRESULT | BuildBTree |
|
|
* Allocates required memory and opens input files to create a B-Tree.
|
|
* Parses incoming words and calls AddRecordToLeaf to process them.
|
|
*
|
|
* @parm HFPB | hfpbSysFile |
|
|
* If not NULL, handle to an already opened sysfile
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to the index parameter block
|
|
*
|
|
* @parm LPB | lpszTemp |
|
|
* Filename of the temporary input file
|
|
*
|
|
* @parm LPB | lpszPerm |
|
|
* Filename of the permanent B-Tree file
|
|
*
|
|
* @rdesc Returns S_OK on success or errors if failed
|
|
*
|
|
*************************************************************************/
|
|
|
|
HRESULT FAR PASCAL BuildBTree (HFPB hfpbFileSys, _LPIPB lpipb,
|
|
LPB lpszTemp, HFPB hfpbPerm, LPSTR lszFilename/*IStream *pistmPerm*/)
|
|
{
|
|
PFILEDATA pOutFile; // Pointer to output data
|
|
PFILEDATA pInFile; // Pointer to input data
|
|
DWORD dwBytesRead = 0; // Checks for EOF
|
|
DWORD dwLeftover; // Used to adjust input buffer
|
|
PBTREEDATA pTreeData = &lpipb->BTreeData; // Structure defining BTree
|
|
PIH20 pHeader = &pTreeData->Header; // Replacement variable
|
|
HRESULT fRet; // Return value
|
|
PNODEINFO pNode; // Pointer to current input node
|
|
ERRB errb= S_OK;
|
|
PHRESULT phr = &errb;
|
|
int iIndex; // Index into the compressed key
|
|
DWORD dwUniqueTerm = 0; // Callback variable
|
|
BOOL fOpenedFile; // TRUE if we have to close the file
|
|
|
|
// Open input file
|
|
pInFile = &lpipb->InFile;
|
|
if ((pInFile->fFile = FileOpen (NULL, lpszTemp,
|
|
REGULAR_FILE, READ, phr)) == NULL)
|
|
return *phr;
|
|
|
|
// Allocate input buffer
|
|
pInFile->dwMax = FILE_BUFFER;
|
|
if ((pInFile->hMem =
|
|
_GLOBALALLOC (DLLGMEM_ZEROINIT, pInFile->dwMax + SAFE_SLACK)) == NULL)
|
|
{
|
|
fRet = E_OUTOFMEMORY;
|
|
exit0:
|
|
FileClose (pInFile->fFile);
|
|
if ((lpipb->idxf & KEEP_TEMP_FILE) == 0)
|
|
FileUnlink (NULL, lpszTemp, REGULAR_FILE);
|
|
return fRet;
|
|
}
|
|
pInFile->pMem = _GLOBALLOCK (pInFile->hMem);
|
|
pInFile->pCurrent = pInFile->pMem;
|
|
|
|
pOutFile = &lpipb->OutFile;
|
|
|
|
/* Open subfile if necessary, (and system file if necessary) */
|
|
pOutFile->fFile = hfpbPerm;
|
|
if ((fOpenedFile = FsTypeFromHfpb(hfpbPerm) != FS_SUBFILE) &&
|
|
(pOutFile->fFile = (HANDLE)FileOpen
|
|
(hfpbPerm, lszFilename, hfpbPerm ? FS_SUBFILE : REGULAR_FILE,
|
|
READ, phr)) == 0)
|
|
{
|
|
SetErrCode (&fRet, E_FILENOTFOUND);
|
|
exit1:
|
|
FreeHandle (pInFile->hMem);
|
|
goto exit0;
|
|
}
|
|
|
|
// Allocate output buffer, at least enough for one block
|
|
pOutFile->dwMax = FILE_BUFFER;
|
|
if (pOutFile->dwMax < (LONG)lpipb->BTreeData.Header.dwBlockSize)
|
|
pOutFile->dwMax = lpipb->BTreeData.Header.dwBlockSize;
|
|
if ((pOutFile->hMem = _GLOBALALLOC (DLLGMEM_ZEROINIT,
|
|
pOutFile->dwMax + SAFE_SLACK)) == NULL)
|
|
{
|
|
fRet = E_OUTOFMEMORY;
|
|
exit2:
|
|
if (fOpenedFile)
|
|
FileClose (hfpbPerm);
|
|
goto exit1;
|
|
}
|
|
pOutFile->pMem = _GLOBALLOCK (pOutFile->hMem);
|
|
|
|
// Skip 1K to hold header infomation
|
|
pOutFile->pCurrent = pOutFile->pMem + FILE_HEADER;
|
|
pOutFile->cbLeft = pOutFile->dwMax - FILE_HEADER;
|
|
pOutFile->foPhysicalOffset.dwOffset = FILE_HEADER;
|
|
pOutFile->ibit = cbitBYTE - 1;
|
|
|
|
// Allocate first leaf node
|
|
if ((pTreeData->rgpNodeInfo[0] = AllocBTreeNode (lpipb)) == NULL)
|
|
{
|
|
fRet = E_OUTOFMEMORY;
|
|
exit3:
|
|
FreeHandle (pOutFile->hMem);
|
|
goto exit2;
|
|
}
|
|
pHeader->nidLast = 1;
|
|
pHeader->cIdxLevels = 1;
|
|
|
|
// pNode points to the leaf node structure
|
|
pNode = pTreeData->rgpNodeInfo[0];
|
|
pNode->Slack = LEAF_SLACK;
|
|
|
|
// Set the bytes left in node block
|
|
pNode->cbLeft = lpipb->BTreeData.Header.dwBlockSize - FOFFSET_SIZE -
|
|
sizeof(WORD);
|
|
|
|
// Set the word length flag
|
|
if (lpipb->occf & OCCF_LENGTH)
|
|
pTreeData->fOccfLength = 1;
|
|
|
|
#if 0
|
|
// Save some math time if we're doing term-weighting
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
MEMSET (pTreeData->argbLog, (BYTE)0, cLOG_MAX * sizeof (BYTE));
|
|
if ((hLog = _GLOBALALLOC (GMEM_MOVEABLE,
|
|
(CB)(cLOG_MAX * sizeof (FLOAT)))) == NULL)
|
|
{
|
|
fRet = E_OUTOFMEMORY;
|
|
goto exit3;
|
|
}
|
|
pTreeData->lrgrLog = (float FAR *)_GLOBALLOCK (hLog);
|
|
}
|
|
else
|
|
hLog = NULL;
|
|
#endif
|
|
|
|
// Load the input buffer & repeat until all records are processed
|
|
pInFile->dwMax = pInFile->cbLeft =
|
|
FileRead (pInFile->fFile, pInFile->pMem, pInFile->dwMax, phr);
|
|
do
|
|
{
|
|
// Call the user callback every once in a while
|
|
if (!(++dwUniqueTerm % 8192L)
|
|
&& (lpipb->CallbackInfo.dwFlags & ERRFLAG_STATUS))
|
|
{
|
|
PFCALLBACK_MSG pCallbackInfo = &lpipb->CallbackInfo;
|
|
CALLBACKINFO Info;
|
|
|
|
Info.dwPhase = 3;
|
|
Info.dwIndex = (DWORD)((float)dwUniqueTerm / lpipb->dwUniqueWord * 100);
|
|
fRet = (*pCallbackInfo->MessageFunc)
|
|
(ERRFLAG_STATUS, pCallbackInfo->pUserData, &Info);
|
|
if (S_OK != fRet)
|
|
goto exit4;
|
|
}
|
|
|
|
if ((fRet = AddRecordToLeaf (lpipb)) != S_OK)
|
|
goto exit4;
|
|
|
|
// pInFile->pCurrent points to the record size
|
|
// 256 is just an arbitrary number of slack to minimize out of data
|
|
|
|
// kevynct: pCurrent points to a record length which does not include
|
|
// the DWORD record len size, so we add this when checking. Actually, we
|
|
// add twice that to be safe.
|
|
if (pInFile->cbLeft <= SAFE_SLACK ||
|
|
(LONG)(GETLONG ((LPUL)(pInFile->pCurrent)) + 2 * sizeof(DWORD)) >= pInFile->cbLeft)
|
|
{
|
|
MEMMOVE (pInFile->pMem, pInFile->pCurrent, pInFile->cbLeft);
|
|
if ((pInFile->cbLeft += FileRead (pInFile->fFile, pInFile->pMem +
|
|
pInFile->cbLeft, pInFile->dwMax - pInFile->cbLeft, phr)) < 0)
|
|
{
|
|
fRet = *phr;
|
|
exit4:
|
|
// Free log block used for term-weighting
|
|
#if 0
|
|
FreeHandle (hLog);
|
|
#endif
|
|
|
|
// Free all node blocks
|
|
dwLeftover = 0;
|
|
while (pTreeData->rgpNodeInfo[dwLeftover] != NULL)
|
|
{
|
|
FreeBTreeNode(pTreeData->rgpNodeInfo[dwLeftover++]);
|
|
}
|
|
goto exit3;
|
|
}
|
|
|
|
pInFile->dwMax = pInFile->cbLeft;
|
|
pInFile->pCurrent = pInFile->pMem;
|
|
}
|
|
} while (fRet == S_OK && pInFile->cbLeft);
|
|
|
|
// Flush anything left in the output buffer
|
|
if ((fRet = FlushAllNodes (lpipb)) != S_OK)
|
|
goto exit4;
|
|
|
|
|
|
// Write out the sigma table
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
pHeader->WeightTabOffset = pOutFile->foPhysicalOffset;
|
|
pHeader->WeightTabSize = (LCB)((lpipb->dwMaxTopicId + 1) *
|
|
sizeof (SIGMA));
|
|
if (FileWrite (pOutFile->fFile, lpipb->wi.hrgsigma,
|
|
pHeader->WeightTabSize, phr) != (LONG)pHeader->WeightTabSize)
|
|
{
|
|
fRet = *phr;
|
|
goto exit4;
|
|
}
|
|
pOutFile->foStartOffset = FoAddDw(pOutFile->foStartOffset,
|
|
pHeader->WeightTabSize);
|
|
}
|
|
|
|
// Copy info to header
|
|
pHeader->FileStamp = INDEX_STAMP;
|
|
pHeader->version = VERCURRENT;
|
|
pHeader->occf = lpipb->occf;
|
|
pHeader->idxf = lpipb->idxf;
|
|
pHeader->lcTopics = lpipb->lcTopics;
|
|
pHeader->dwMaxTopicId = lpipb->dwMaxTopicId;
|
|
pHeader->dwMaxFieldId = lpipb->dwMaxFieldId;
|
|
|
|
pHeader->dwMaxWCount = lpipb->dwMaxWCount;
|
|
pHeader->dwMaxOffset = lpipb->dwMaxOffset;
|
|
pHeader->dwMaxWLen = lpipb->dwMaxWLen;
|
|
pHeader->dwTotalWords = lpipb->dwIndexedWord; // Total indexed words
|
|
pHeader->dwUniqueWords = lpipb->dwUniqueWord; // Total unique words
|
|
pHeader->dwTotal2bWordLen = lpipb->dwTotal2bWordLen;
|
|
pHeader->dwTotal3bWordLen = lpipb->dwTotal3bWordLen;
|
|
pHeader->dwUniqueWordLen = lpipb->dwTotalUniqueWordLen;
|
|
|
|
pHeader->ckeyTopicId = lpipb->cKey[CKEY_TOPIC_ID];
|
|
pHeader->ckeyOccCount = lpipb->cKey[CKEY_OCC_COUNT];
|
|
iIndex = CKEY_OCC_BASE;
|
|
if (pHeader->occf & OCCF_COUNT)
|
|
pHeader->ckeyWordCount = lpipb->cKey[iIndex++];
|
|
if (pHeader->occf & OCCF_OFFSET)
|
|
pHeader->ckeyOffset = lpipb->cKey[iIndex];
|
|
|
|
if (FileSeekWrite (pOutFile->fFile, (LPB)pHeader, MakeFo (0, 0),
|
|
sizeof (IH20), phr) != sizeof (IH20))
|
|
{
|
|
fRet = *phr;
|
|
goto exit4;
|
|
}
|
|
|
|
// Call the user callback every once in a while
|
|
if (lpipb->CallbackInfo.dwFlags & ERRFLAG_STATUS)
|
|
{
|
|
PFCALLBACK_MSG pCallbackInfo = &lpipb->CallbackInfo;
|
|
CALLBACKINFO Info;
|
|
|
|
Info.dwPhase = 3;
|
|
Info.dwIndex = 100;
|
|
fRet = (*pCallbackInfo->MessageFunc)
|
|
(ERRFLAG_STATUS, pCallbackInfo->pUserData, &Info);
|
|
if (S_OK != fRet)
|
|
goto exit4;
|
|
}
|
|
fRet = S_OK;
|
|
goto exit4;
|
|
} /* BuildBTree */
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func HRESULT | AddRecordToLeaf |
|
|
* Add the record pointed to by pDtreeData->OutFile->pCurrent to the B-Tree
|
|
* contained in the structure.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to the index parameter block
|
|
*
|
|
* @rdesc Returns S_OK on success or errors if failed
|
|
*
|
|
*************************************************************************/
|
|
#ifdef _DEBUG
|
|
static BYTE LastWord[4000] = {0};
|
|
static BYTE CurWord[4000] = {0};
|
|
#endif
|
|
|
|
HRESULT PASCAL AddRecordToLeaf (_LPIPB lpipb)
|
|
{
|
|
// Local Replacement Variables
|
|
PBTREEDATA pTreeData = &lpipb->BTreeData;
|
|
PFILEDATA pOutFile = &lpipb->OutFile; // Output data
|
|
PFILEDATA pInFile = &lpipb->InFile; // Input data
|
|
HFPB fOutput = pOutFile->fFile; // Output file
|
|
HFPB fInput = lpipb->InFile.fFile; // Input file
|
|
LPB pInCurPtr = lpipb->InFile.pCurrent; // Input buffer
|
|
PNODEINFO pNode;
|
|
LPB lpbWord; // Pointer to the word string
|
|
OCCF occf = lpipb->occf;
|
|
|
|
// Working Variables
|
|
DWORD dwTopicCount; // Number of topic in record
|
|
DWORD dwFieldId;
|
|
DWORD dwBlockSize; // Size of the entire occ block
|
|
LPB pDest;
|
|
WORD uStringSize;
|
|
ERRB errb;
|
|
|
|
|
|
// We always start from the leaf node
|
|
pNode = pTreeData->rgpNodeInfo[0];
|
|
|
|
// Set pointer to working buffer
|
|
pDest = pNode->pTmpResult;
|
|
|
|
// Advance input buffer to the word string
|
|
pInCurPtr += sizeof (DWORD);
|
|
lpbWord = pInCurPtr;
|
|
|
|
// Insert the word into the buffer
|
|
pDest += PrefixCompressWord (pDest, pInCurPtr,
|
|
pNode->pLastWord, pTreeData->fOccfLength);
|
|
|
|
// Get the word length
|
|
uStringSize = GETWORD((LPUW)pInCurPtr);
|
|
lpipb->dwTotalUniqueWordLen += uStringSize;
|
|
|
|
// Adjust for the word length storage
|
|
uStringSize += sizeof(SHORT);
|
|
|
|
// Skip the word
|
|
pInCurPtr += uStringSize;
|
|
|
|
#ifdef _DEBUG
|
|
STRCPY (LastWord, CurWord);
|
|
MEMCPY (CurWord, lpbWord + 2, GETWORD((LPUW)lpbWord));
|
|
CurWord[GETWORD((LPUW)lpbWord)] = 0;
|
|
if (STRCMP (LastWord, CurWord) > 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
// if (STRCMP (CurWord, "forbidden") == 0)
|
|
// _asm int 3;
|
|
#endif
|
|
|
|
// If OccfLength is set skip it now
|
|
// (It has already been appended to the compressed word)
|
|
if (pTreeData->fOccfLength)
|
|
pInCurPtr += CbByteUnpack(&dwBlockSize, pInCurPtr);
|
|
|
|
// Copy the FieldID
|
|
if (occf & OCCF_FIELDID)
|
|
{
|
|
CbByteUnpack (&dwFieldId, pInCurPtr);
|
|
do {
|
|
*pDest++ = *pInCurPtr;
|
|
} while (*pInCurPtr++ & 0x80);
|
|
}
|
|
|
|
// Get Topic Count
|
|
#if 0
|
|
CbByteUnpack (&dwTopicCount, pInCurPtr);
|
|
do
|
|
{
|
|
*pDest++ = *pInCurPtr;
|
|
} while (*pInCurPtr++ & 0x80);
|
|
#else
|
|
dwTopicCount = GETLONG((LPUL)pInCurPtr);
|
|
pInCurPtr += sizeof(DWORD);
|
|
pDest += CbBytePack(pDest, dwTopicCount);
|
|
#endif
|
|
|
|
// Check to see if this entry will fit in the leaf node
|
|
// We can't write the data block until we know where the entry
|
|
// will be stored. We must add in FOFFSET_SIZE to our current location
|
|
// to determine size. We ignore the block size field, so we might encroach
|
|
// on the slack by a few bytes.
|
|
if (pNode->cbLeft - pNode->Slack < (SHORT)(pDest -pNode->pTmpResult +FOFFSET_SIZE))
|
|
{
|
|
HRESULT fRet;
|
|
|
|
if ((fRet = AddRecordToStem (lpipb, lpbWord)) != S_OK)
|
|
return(fRet);
|
|
|
|
// If the prefix count is zero, no problem
|
|
// Else we have to re-copy the word, since we are in a new leaf node
|
|
if (0 != pNode->pTmpResult[1])
|
|
{
|
|
dwBlockSize = PrefixCompressWord (pNode->pTmpResult, lpbWord,
|
|
pNode->pLastWord, pTreeData->fOccfLength);
|
|
pDest = pNode->pTmpResult + dwBlockSize;
|
|
if (occf & OCCF_FIELDID)
|
|
pDest += CbBytePack (pDest, dwFieldId);
|
|
pDest += CbBytePack (pDest, dwTopicCount);
|
|
}
|
|
}
|
|
// Save new word as last word
|
|
MEMCPY (pNode->pLastWord, lpbWord, uStringSize + 2);
|
|
|
|
// Set pointer to beginning of data block
|
|
pDest += CopyFileOffset (pDest, pOutFile->foPhysicalOffset);
|
|
|
|
// Update the bytes left
|
|
pInFile->cbLeft -= (LONG) (pInCurPtr - pInFile->pCurrent);
|
|
#ifdef _DEBUG
|
|
if (pInFile->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
|
|
// Compress data block to output buffer and store it's compressed size
|
|
pInFile->pCurrent = pInCurPtr;
|
|
if ((dwBlockSize = WriteDataNode (lpipb, dwTopicCount, &errb)) == 0)
|
|
return errb;
|
|
pDest += CbBytePack (pDest, dwBlockSize);
|
|
|
|
// Copy the temp buffer to the real node
|
|
dwBlockSize = (DWORD)(pDest - pNode->pTmpResult);
|
|
MEMCPY (pNode->pCurPtr, pNode->pTmpResult, dwBlockSize);
|
|
pNode->pCurPtr += dwBlockSize;
|
|
pNode->cbLeft -= (WORD)dwBlockSize;
|
|
|
|
return S_OK;
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func DWORD | AddRecordToStem |
|
|
* Add a key to a stem node, creating/flushing nodes as necessary.
|
|
*
|
|
* @parm LPB | lpbWord |
|
|
* The word to add the the stem node (last word in the full leaf node)
|
|
*
|
|
* @rdesc S_OK if successful, or errors if failed
|
|
*
|
|
*************************************************************************/
|
|
|
|
HRESULT PASCAL AddRecordToStem (_LPIPB lpipb, LPB lpbWord)
|
|
{
|
|
SHORT CurLevel = 0;
|
|
PNODEINFO pStemNode;
|
|
PNODEINFO pLastNode;
|
|
|
|
PBTREEDATA pTreeData = &lpipb->BTreeData;
|
|
PNODEINFO pLeafNode = pTreeData->rgpNodeInfo[0];
|
|
LPB pLastWord;
|
|
int cbTemp;
|
|
ERRB errb = S_OK;
|
|
HRESULT fRet;
|
|
|
|
// Move up through stem nodes until space can be found/made
|
|
pStemNode = pLeafNode;
|
|
do
|
|
{
|
|
pLastWord = pStemNode->pLastWord;
|
|
pStemNode = pTreeData->rgpNodeInfo[++CurLevel];
|
|
if (pStemNode == NULL)
|
|
{ // Create a new stem node
|
|
if ((pStemNode = pTreeData->rgpNodeInfo[CurLevel] =
|
|
AllocBTreeNode (lpipb)) == NULL)
|
|
return SetErrCode (NULL, E_OUTOFMEMORY);
|
|
pStemNode->Slack = STEM_SLACK;
|
|
pStemNode->cbLeft = lpipb->BTreeData.Header.dwBlockSize
|
|
- sizeof(WORD);
|
|
if (++pTreeData->Header.cIdxLevels > MAX_TREE_HEIGHT)
|
|
return E_TREETOOBIG;
|
|
}
|
|
pTreeData->Header.nidLast++;
|
|
} while (pStemNode->cbLeft - pStemNode->Slack <
|
|
(SHORT)(GETWORD ((LPUW)pLastWord) + sizeof (SHORT) + FOFFSET_SIZE));
|
|
|
|
// Work back down through the nodes clearing them to disk
|
|
while (CurLevel > 1)
|
|
{
|
|
pLastNode = pTreeData->rgpNodeInfo[--CurLevel];
|
|
pLastWord = pLastNode->pLastWord;
|
|
// Copy word to stem node
|
|
if ((cbTemp = PrefixCompressWord (pStemNode->pCurPtr, pLastWord,
|
|
pStemNode->pLastWord, pTreeData->fOccfLength)) == 0)
|
|
{
|
|
return errb;
|
|
}
|
|
pStemNode->pCurPtr += cbTemp;
|
|
|
|
// Update the last word in the stem node
|
|
MEMCPY (pStemNode->pLastWord, pLastWord, GETWORD((LPUW)pLastWord)+ 2*sizeof(WORD));
|
|
|
|
// Set pointer in stem node
|
|
CopyFileOffset (pStemNode->pCurPtr,
|
|
lpipb->OutFile.foPhysicalOffset);
|
|
pStemNode->pCurPtr += FOFFSET_SIZE;
|
|
pStemNode->cbLeft -= FOFFSET_SIZE + cbTemp;
|
|
#ifdef _DEBUG
|
|
if (pStemNode->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
|
|
pStemNode = pTreeData->rgpNodeInfo[CurLevel];
|
|
if ((fRet = WriteStemNode (lpipb, pStemNode)) != S_OK)
|
|
return(fRet);
|
|
|
|
}
|
|
|
|
// Clear the leaf node into the first stem node & reset it
|
|
|
|
// Copy last word to stem node
|
|
if ((cbTemp = PrefixCompressWord (pStemNode->pCurPtr,
|
|
pLeafNode->pLastWord, pStemNode->pLastWord,
|
|
pTreeData->fOccfLength)) == 0)
|
|
{
|
|
return errb;
|
|
}
|
|
pStemNode->pCurPtr += cbTemp;
|
|
pStemNode->cbLeft -= cbTemp;
|
|
#ifdef _DEBUG
|
|
if (pStemNode->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
|
|
|
|
// Update the last word in the stem node
|
|
MEMCPY (pStemNode->pLastWord, pLeafNode->pLastWord,
|
|
GETWORD((LPUW)(pLeafNode->pLastWord))+2*sizeof(WORD));
|
|
|
|
// Set pointer to the leaf node
|
|
CopyFileOffset (pStemNode->pCurPtr, lpipb->OutFile.foPhysicalOffset);
|
|
pStemNode->pCurPtr += FOFFSET_SIZE;
|
|
pStemNode->cbLeft -= FOFFSET_SIZE;
|
|
#ifdef _DEBUG
|
|
if (pStemNode->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
|
|
// Flush leaf node to output buffer and reset it
|
|
return WriteLeafNode (lpipb);
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func int | CompressDword |
|
|
* Compresses the input stream into the output buffer using a high
|
|
* bit encoding method. If the buffer is full it will be flushed to
|
|
* a file.
|
|
*
|
|
* @parm PFILEDATA | pOutput |
|
|
* Pointer to output buffer info
|
|
*
|
|
* @parm LPDWORD | pSrc |
|
|
* Pointer to the uncompressed input stream
|
|
*
|
|
* @rdesc Returns the number of compressed bytes buffered
|
|
*
|
|
*************************************************************************/
|
|
|
|
int PASCAL CompressDword (PFILEDATA pOutput, DWORD dwValue)
|
|
{
|
|
LPB pDest = pOutput->pCurrent;
|
|
int cBytes = 0; // Count of compressed bytes
|
|
ERRB errb;
|
|
|
|
// Any room left in output buffer?
|
|
if (sizeof(DWORD) * 2 >= pOutput->cbLeft)
|
|
{
|
|
DWORD dwSize;
|
|
|
|
FileWrite (pOutput->fFile, pOutput->pMem,
|
|
(dwSize = (DWORD)(pDest - pOutput->pMem)), &errb);
|
|
pDest = pOutput->pMem;
|
|
pOutput->cbLeft = pOutput->dwMax;
|
|
pOutput->foStartOffset = FoAddDw(pOutput->foStartOffset, dwSize);
|
|
}
|
|
|
|
while (dwValue)
|
|
{
|
|
*pDest = (BYTE)(dwValue & 0x7F);
|
|
cBytes++;
|
|
dwValue >>= 7;
|
|
if (dwValue != 0)
|
|
*pDest |= 0x80;
|
|
pDest++;
|
|
}
|
|
pOutput->pCurrent = pDest;
|
|
pOutput->foPhysicalOffset =
|
|
FoAddDw (pOutput->foPhysicalOffset, (DWORD)cBytes);
|
|
pOutput->cbLeft -= cBytes;
|
|
#ifdef _DEBUG
|
|
if (pOutput->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
return cBytes;
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func DWORD | WriteDataNode |
|
|
* Compresses the input stream into the output buffer. If the buffer
|
|
* is full it will be flushed to a file.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to global buffer
|
|
*
|
|
* @parm DWORD | dwTopicCount |
|
|
* The number of topics in the input stream
|
|
*
|
|
* @parm PHRESULT | phr |
|
|
* Error buffer
|
|
*
|
|
* @rdesc Returns the number of compressed bytes written
|
|
*
|
|
*************************************************************************/
|
|
|
|
PUBLIC DWORD PASCAL FAR WriteDataNode (_LPIPB lpipb,
|
|
DWORD dwTopicCount, PHRESULT phr)
|
|
{
|
|
// Local replacement Variables
|
|
PBTREEDATA pTreeData = &lpipb->BTreeData;
|
|
PFILEDATA pOutput = &lpipb->OutFile; // Output data structure
|
|
PFILEDATA pInFile = &lpipb->InFile; // Input data structre
|
|
HFPB fFile = pOutput->fFile; // Output file handle
|
|
|
|
// Working Variables
|
|
DWORD dwBlockSize; // Size of block to compress
|
|
DWORD dwEncodedSize = 0; // Size of encoded block
|
|
DWORD dwTopicIdDelta; // Really only used for weight values
|
|
DWORD TopicLoop;
|
|
DWORD dwSlackSize;
|
|
DWORD loop;
|
|
DWORD dwTemp;
|
|
FILEOFFSET foStart; // Physical beginning of bit compression block
|
|
FLOAT rTerm; // Only used when IDXF_NORMALIZE is set
|
|
FLOAT rWeight; // Only used when IDXF_NORMALIZE is set
|
|
WORD wWeight; // Only used when IDXF_NORMALIZE is set
|
|
DWORD dwTopicId = 0; // Only used when IDXF_NORMALIZE is set
|
|
int cbTemp; // # of compressed bytes that uncompressed
|
|
OCCF occf = lpipb->occf;
|
|
HRESULT fRet;
|
|
|
|
foStart = pOutput->foPhysicalOffset;
|
|
wWeight = 0; // UNDONE: Don't need it
|
|
|
|
for (TopicLoop = dwTopicCount; TopicLoop > 0; --TopicLoop)
|
|
{
|
|
// Move to the byte boundary
|
|
if (pOutput->ibit != cbitBYTE - 1)
|
|
{
|
|
pOutput->ibit = cbitBYTE - 1;
|
|
|
|
if (--pOutput->cbLeft)
|
|
{
|
|
pOutput->pCurrent++;
|
|
pOutput->foPhysicalOffset = FoAddDw (pOutput->foPhysicalOffset, 1);
|
|
}
|
|
else
|
|
{
|
|
if (FileWrite (pOutput->fFile, pOutput->pMem,
|
|
dwTemp = (DWORD)(pOutput->pCurrent - pOutput->pMem), phr) != (LONG)dwTemp)
|
|
return(0);
|
|
pOutput->pCurrent = pOutput->pMem;
|
|
pOutput->cbLeft = pOutput->dwMax;
|
|
pOutput->foStartOffset = FoAddDw(pOutput->foStartOffset, dwTemp);
|
|
#ifdef _DEBUG
|
|
MEMSET (pOutput->pMem, 0, pOutput->dwMax);
|
|
#endif
|
|
}
|
|
}
|
|
// Store TopicId as necessary
|
|
if (pInFile->cbLeft < 2 * sizeof (DWORD))
|
|
{
|
|
MEMMOVE (pInFile->pMem, pInFile->pCurrent, pInFile->cbLeft);
|
|
pInFile->cbLeft += FileRead (pInFile->fFile, pInFile->pMem + pInFile->cbLeft,
|
|
pInFile->dwMax - pInFile->cbLeft, phr);
|
|
pInFile->dwMax = pInFile->cbLeft;
|
|
pInFile->pCurrent = pInFile->pMem;
|
|
}
|
|
cbTemp = CbByteUnpack (&dwTopicIdDelta, pInFile->pCurrent);
|
|
dwTopicId += dwTopicIdDelta; // Get the real TopicID
|
|
if ((fRet = FAddDword (pOutput, dwTopicIdDelta,
|
|
lpipb->cKey[CKEY_TOPIC_ID])) != S_OK)
|
|
{
|
|
SetErrCode(phr, fRet);
|
|
return(0);
|
|
|
|
}
|
|
|
|
pInFile->pCurrent += cbTemp;
|
|
pInFile->cbLeft -= cbTemp;
|
|
|
|
if (occf & OCCF_HAVE_OCCURRENCE)
|
|
{
|
|
|
|
// Get number of occ data records for this topic
|
|
if (pInFile->cbLeft < 2 * sizeof (DWORD))
|
|
{
|
|
MEMMOVE (pInFile->pMem, pInFile->pCurrent, pInFile->cbLeft);
|
|
pInFile->cbLeft += FileRead (pInFile->fFile,
|
|
pInFile->pMem + pInFile->cbLeft,
|
|
pInFile->dwMax - pInFile->cbLeft, phr);
|
|
pInFile->dwMax = pInFile->cbLeft;
|
|
pInFile->pCurrent = pInFile->pMem;
|
|
}
|
|
cbTemp = CbByteUnpack (&dwBlockSize, pInFile->pCurrent);
|
|
pInFile->pCurrent += cbTemp;
|
|
pInFile->cbLeft -= cbTemp;
|
|
}
|
|
|
|
// If we are term weighing we have to calculate the weight
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
#ifndef ISBU_IR_CHANGE
|
|
// log10(x/y) == log10 (x) - log10 (y). Since x in our case is a known constant,
|
|
// 100,000,000, I'm replacing that with its equivalent log10 value of 8.0 and subtracting
|
|
// the log10(y) from it
|
|
rTerm = (float) (8.0 - log10((double) dwTopicCount));
|
|
// In extreme cases, rTerm could be 0 or even -ve (when dwTopicCount approaches or
|
|
// exceeds 100,000,000)
|
|
if (rTerm <= (float) 0.0)
|
|
rTerm = cVerySmallWt; // very small value. == log(100 mil/ 95 mil)
|
|
// NOTE : rWeight for the doc term would be as follows:
|
|
// rWeight = float(min(4096, dwBlockSize)) * rTerm / lpipb->wi.hrgsigma[dwTopicId]
|
|
//
|
|
// Since rTerm needs to be recomputed again for the query term weight computation,
|
|
// and since rTerm will be the same value for the current term ('cos N and n of log(N/n)
|
|
// are the same (N = 100 million and n is whatever the doc term freq is for the term),
|
|
// we will factor in the second rTerm at index time. This way, we don't have to deal
|
|
// with rTerm at search time (reduces computation and query time shortens)
|
|
//
|
|
// MV 2.0 initially did the same thing. However, BinhN removed the second rTerm
|
|
// because he decided to remove the rTerm altogether from the query term weight. He
|
|
// did that to keep the scores reasonably high.
|
|
|
|
rWeight = ((float) min(cTFThreshold, dwBlockSize)) * rTerm * rTerm / lpipb->wi.hrgsigma[dwTopicId];
|
|
// without the additional rTerm, we would probably be between 0.0 and 1.0
|
|
if (rWeight > rTerm)
|
|
wWeight = 0xFFFF;
|
|
else
|
|
wWeight = (WORD) ((float)0xFFFF * rWeight / rTerm);
|
|
#else
|
|
rTerm = (float) (65535.0 * 8) / (float)dwTopicCount;
|
|
|
|
rWeight = (float)dwBlockSize * rTerm / lpipb->wi.hrgsigma[dwTopicId];
|
|
if (rWeight >= 65535.0)
|
|
wWeight = 65335;
|
|
else
|
|
wWeight = (WORD)rWeight;
|
|
#endif // ISBU_IR_CHANGE
|
|
|
|
// Write the weight to the output buffer
|
|
if ((fRet = FWriteBits (&lpipb->OutFile, (DWORD)wWeight,
|
|
(BYTE)(sizeof (WORD) * cbitBYTE))) != S_OK)
|
|
{
|
|
SetErrCode (phr, fRet);
|
|
return(0);
|
|
}
|
|
}
|
|
|
|
// Don't do anything else if there is nothing else to do!!!
|
|
if ((occf & OCCF_HAVE_OCCURRENCE) == 0)
|
|
continue;
|
|
|
|
// Write the OccCount
|
|
if ((fRet = FAddDword (pOutput, dwBlockSize,
|
|
lpipb->cKey[CKEY_OCC_COUNT])) != S_OK)
|
|
{
|
|
SetErrCode (phr, fRet);
|
|
return(0);
|
|
}
|
|
|
|
// Encode the occ block
|
|
for (loop = dwBlockSize; loop > 0; loop--)
|
|
{
|
|
int iIndex;
|
|
|
|
iIndex = CKEY_OCC_BASE;
|
|
|
|
// Make sure input buffer holds enough data
|
|
if (pInFile->cbLeft < 5 * sizeof (DWORD))
|
|
{
|
|
MEMMOVE (pInFile->pMem, pInFile->pCurrent, pInFile->cbLeft);
|
|
pInFile->cbLeft += FileRead (pInFile->fFile,
|
|
pInFile->pMem + pInFile->cbLeft,
|
|
pInFile->dwMax - pInFile->cbLeft, phr);
|
|
pInFile->dwMax = pInFile->cbLeft;
|
|
pInFile->pCurrent = pInFile->pMem;
|
|
}
|
|
|
|
if (occf & OCCF_COUNT)
|
|
{
|
|
cbTemp = CbByteUnpack (&dwTemp, pInFile->pCurrent);
|
|
pInFile->pCurrent += cbTemp;
|
|
pInFile->cbLeft -= cbTemp;
|
|
if ((fRet = FAddDword (pOutput, dwTemp, lpipb->cKey[iIndex])) !=
|
|
S_OK)
|
|
{
|
|
SetErrCode (phr, fRet);
|
|
return(0);
|
|
}
|
|
iIndex++;
|
|
}
|
|
if (occf & OCCF_OFFSET)
|
|
{
|
|
cbTemp = CbByteUnpack (&dwTemp, pInFile->pCurrent);
|
|
pInFile->pCurrent += cbTemp;
|
|
pInFile->cbLeft -= cbTemp;
|
|
if ((fRet = FAddDword (pOutput, dwTemp, lpipb->cKey[iIndex])) !=
|
|
S_OK)
|
|
{
|
|
SetErrCode (phr, fRet);
|
|
return(0);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Advance to next byte (we are partially through a byte now)
|
|
pOutput->ibit = cbitBYTE - 1;
|
|
pOutput->pCurrent++;
|
|
pOutput->foPhysicalOffset = FoAddDw (pOutput->foPhysicalOffset, 1);
|
|
pOutput->cbLeft--;
|
|
#ifdef _DEBUG
|
|
if (pOutput->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
dwEncodedSize += DwSubFo (pOutput->foPhysicalOffset, foStart);
|
|
|
|
// Leave slack space, but not for uncommon words
|
|
if (dwTopicCount <= 2)
|
|
dwSlackSize = 0;
|
|
else
|
|
dwSlackSize = dwEncodedSize / 10;
|
|
|
|
dwEncodedSize += dwSlackSize;
|
|
// Keep a running total of all allocated slack space
|
|
pTreeData->Header.dwSlackCount += dwSlackSize;
|
|
|
|
while (dwSlackSize)
|
|
{
|
|
if (pOutput->cbLeft < (LONG)dwSlackSize)
|
|
{ // The slack block doesn't fit in the output buffer
|
|
// Write as much as we can then flush the buffer and write the rest
|
|
// MEMSET (pOutput->pCurrent, 0, pOutput->cbLeft);
|
|
DWORD dwSize;
|
|
|
|
dwSlackSize -= pOutput->cbLeft;
|
|
if (0 == FileWrite (fFile, pOutput->pMem,
|
|
dwSize = pOutput->dwMax, phr))
|
|
{
|
|
return 0;
|
|
}
|
|
pOutput->pCurrent = pOutput->pMem;
|
|
pOutput->foPhysicalOffset =
|
|
FoAddDw (pOutput->foPhysicalOffset, pOutput->cbLeft);
|
|
pOutput->cbLeft = pOutput->dwMax;
|
|
pOutput->foStartOffset = FoAddDw(pOutput->foStartOffset, dwSize);
|
|
}
|
|
else
|
|
{ // The slack fits, no problems
|
|
MEMSET (pOutput->pCurrent, 0, dwSlackSize);
|
|
pOutput->pCurrent += dwSlackSize;
|
|
pOutput->foPhysicalOffset =
|
|
FoAddDw (pOutput->foPhysicalOffset, dwSlackSize);
|
|
pOutput->cbLeft -= dwSlackSize;
|
|
#ifdef _DEBUG
|
|
if (pOutput->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
dwSlackSize = 0;
|
|
}
|
|
}
|
|
return dwEncodedSize;
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func void | WriteStemNode |
|
|
* Flushes a stem node in the BTree to the output buffer. Once flushed,
|
|
* the node is reset to the beginning and filled with zeros.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer the IPB structure
|
|
*
|
|
* @parm PNODEINFO | pNode |
|
|
* Pointer to the node to flush
|
|
*
|
|
*************************************************************************/
|
|
|
|
PRIVATE HRESULT PASCAL WriteStemNode (_LPIPB lpipb, PNODEINFO pNode)
|
|
{
|
|
// Local Replacement Variable
|
|
PBTREEDATA pTreeData = &lpipb->BTreeData;
|
|
PFILEDATA pOutput = &lpipb->OutFile; // Output structure
|
|
LPB pDest; // Output buffer
|
|
LPB pStart = pNode->pBuffer; // Start of node buffer
|
|
|
|
// Local Working Variables
|
|
DWORD dwBytesLeft; // Bytes left to write
|
|
ERRB errb;
|
|
|
|
#if 0 // Use 2-bytes for cbLeft to simplify the work of update
|
|
// Compress CbLeft to output buffer
|
|
dwBytesLeft = lpipb->BTreeData.Header.dwBlockSize - FOFFSET_SIZE -
|
|
CompressDword (pOutput, (DWORD)pNode->cbLeft);
|
|
#else
|
|
*(LPUW)(pOutput->pCurrent) = (WORD)pNode->cbLeft;
|
|
pOutput->pCurrent += sizeof(WORD);
|
|
pOutput->cbLeft -= sizeof(WORD);
|
|
pOutput->foPhysicalOffset =
|
|
FoAddDw (pOutput->foPhysicalOffset, (DWORD)sizeof(WORD));
|
|
dwBytesLeft = lpipb->BTreeData.Header.dwBlockSize - sizeof(WORD);
|
|
#endif
|
|
pDest = pOutput->pCurrent;
|
|
|
|
// Keep a running total of all allocated slack space
|
|
pTreeData->Header.dwSlackCount += pNode->cbLeft;
|
|
|
|
// This is why the buffer must be >= BTREE_NODE_SIZE
|
|
// This could be put in a loop to avoid that restriction, but it
|
|
// is probably not worth it. (See also WriteLeafNode)
|
|
if (pOutput->cbLeft < (LONG)dwBytesLeft)
|
|
{
|
|
LONG dwSize;
|
|
if (FileWrite (pOutput->fFile,
|
|
pOutput->pMem, dwSize = (DWORD)(pDest - pOutput->pMem), &errb) != dwSize)
|
|
return(errb);
|
|
|
|
pDest = pOutput->pMem;
|
|
pOutput->cbLeft = pOutput->dwMax;
|
|
pOutput->foStartOffset = FoAddDw(pOutput->foStartOffset, dwSize);
|
|
}
|
|
MEMCPY (pDest, pStart, dwBytesLeft);
|
|
pOutput->foPhysicalOffset =
|
|
FoAddDw (pOutput->foPhysicalOffset, dwBytesLeft);
|
|
pOutput->cbLeft -= dwBytesLeft;
|
|
#ifdef _DEBUG
|
|
if (pOutput->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
// Set the external variable
|
|
pOutput->pCurrent = pDest + dwBytesLeft;
|
|
|
|
// Set to all zeros so we know when we have reached the end of data later
|
|
MEMSET (pNode->pBuffer, 0, lpipb->BTreeData.Header.dwBlockSize);
|
|
pNode->cbLeft = lpipb->BTreeData.Header.dwBlockSize - sizeof(WORD);
|
|
pNode->pCurPtr = pNode->pBuffer;
|
|
*(PUSHORT)pNode->pLastWord = 0;
|
|
return(S_OK);
|
|
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func void | WriteLeafNode |
|
|
* Flushes a leaf node in the BTree to the output buffer. Once flushed,
|
|
* the node is reset to the beginning and filled with zeros.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to index block
|
|
*
|
|
* @rdesc S_OK or other errors
|
|
*************************************************************************/
|
|
|
|
PRIVATE HRESULT PASCAL NEAR WriteLeafNode (_LPIPB lpipb)
|
|
{
|
|
// Local Replacement Variables
|
|
PBTREEDATA pTreeData = &lpipb->BTreeData;
|
|
PFILEDATA pOutput = &lpipb->OutFile; // Output data structure
|
|
LPB pDest = pOutput->pCurrent; // Output buffer
|
|
FILEOFFSET OffsetPointer = pTreeData->OffsetPointer;
|
|
FILEOFFSET foPhysicalOffset = pOutput->foPhysicalOffset;
|
|
PNODEINFO pNode = pTreeData->rgpNodeInfo[0]; // Leaf node
|
|
LPB pStart = pNode->pBuffer; // Beginning of the node buffer
|
|
// Working Variables
|
|
DWORD dwLeft;
|
|
FILEOFFSET StartOffset; // Physical offset of the begining
|
|
// of the output buffer
|
|
ERRB errb;
|
|
|
|
// Backpatch the current offset to the last nodes pointer
|
|
if (!FoIsNil (OffsetPointer))
|
|
{
|
|
// Is the backpatch location in the output buffer?
|
|
if (FoCompare (OffsetPointer,
|
|
(StartOffset = FoSubFo (foPhysicalOffset,
|
|
MakeFo ((DWORD)(pDest - pOutput->pMem), 0)))) >= 0)
|
|
{
|
|
CopyFileOffset (pOutput->pMem + DwSubFo
|
|
(OffsetPointer, StartOffset), foPhysicalOffset);
|
|
}
|
|
else
|
|
{
|
|
if (FileSeekWrite (pOutput->fFile, &foPhysicalOffset,
|
|
OffsetPointer, sizeof (DWORD), &errb) !=
|
|
sizeof (DWORD))
|
|
return(errb);
|
|
|
|
FileSeek (pOutput->fFile, StartOffset, 0, NULL);
|
|
}
|
|
}
|
|
// Set the backpatch location for next time
|
|
pTreeData->OffsetPointer = foPhysicalOffset;
|
|
|
|
// Skip the record pointer for this record (will be backpatched next time)
|
|
|
|
if (pOutput->cbLeft <= 0 )
|
|
{
|
|
LONG dwSize;
|
|
|
|
if (FileWrite (pOutput->fFile, pOutput->pMem,
|
|
dwSize = (DWORD)(pDest - pOutput->pMem), &errb) != dwSize)
|
|
return(errb);
|
|
|
|
pDest = pOutput->pMem;
|
|
pOutput->cbLeft = pOutput->dwMax;
|
|
pOutput->foStartOffset = FoAddDw(pOutput->foStartOffset, dwSize);
|
|
}
|
|
MEMSET (pDest, 0, FOFFSET_SIZE);
|
|
pOutput->cbLeft -= FOFFSET_SIZE;
|
|
#ifdef _DEBUG
|
|
if (pOutput->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
pOutput->pCurrent = pDest + FOFFSET_SIZE;
|
|
pOutput->foPhysicalOffset = FoAddDw (foPhysicalOffset, FOFFSET_SIZE);
|
|
|
|
#if 0 // Use 2-bytes for cbLeft to simplify the work of update
|
|
// Compress CbLeft to output buffer
|
|
dwLeft = lpipb->BTreeData.Header.dwBlockSize - FOFFSET_SIZE -
|
|
CompressDword (pOutput, (DWORD)pNode->cbLeft);
|
|
#else
|
|
*(LPUW)(pOutput->pCurrent) = (WORD)pNode->cbLeft;
|
|
pOutput->foPhysicalOffset =
|
|
FoAddDw (pOutput->foPhysicalOffset, (DWORD)sizeof(WORD));
|
|
pOutput->cbLeft -= sizeof(WORD);
|
|
dwLeft = lpipb->BTreeData.Header.dwBlockSize - FOFFSET_SIZE - sizeof(WORD);
|
|
pOutput->pCurrent += sizeof(WORD);
|
|
#endif
|
|
pDest = pOutput->pCurrent;
|
|
|
|
// Keep a running total of all allocated slack space
|
|
pTreeData->Header.dwSlackCount += pNode->cbLeft;
|
|
|
|
// This is why the buffer must be >= BTREE_NODE_SIZE
|
|
// This could be put in a loop to avoid that restriction, but it
|
|
// is probably not worth it. (See also WriteStemNode)
|
|
if (pOutput->cbLeft < (LONG)dwLeft)
|
|
{
|
|
LONG dwSize;
|
|
if (FileWrite (pOutput->fFile, pOutput->pMem,
|
|
dwSize = (DWORD)(pDest - pOutput->pMem), &errb) != dwSize)
|
|
return(errb);
|
|
|
|
pDest = pOutput->pMem;
|
|
pOutput->cbLeft = pOutput->dwMax;
|
|
pOutput->foStartOffset = FoAddDw(pOutput->foStartOffset, dwSize);
|
|
}
|
|
MEMCPY (pDest, pStart, dwLeft);
|
|
pOutput->foPhysicalOffset =
|
|
FoAddDw (pOutput->foPhysicalOffset, dwLeft);
|
|
pOutput->cbLeft -= dwLeft;
|
|
#ifdef _DEBUG
|
|
if (pOutput->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
pOutput->pCurrent = pDest + dwLeft;
|
|
|
|
// Reset buffer back to beginning
|
|
MEMSET (pNode->pBuffer, 0, lpipb->BTreeData.Header.dwBlockSize);
|
|
pNode->pCurPtr = pNode->pBuffer;
|
|
|
|
// Set the bytes left in node block
|
|
pNode->cbLeft = lpipb->BTreeData.Header.dwBlockSize -
|
|
FOFFSET_SIZE - sizeof(WORD);
|
|
*(PUSHORT)pNode->pLastWord = 0;
|
|
return(S_OK);
|
|
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func PNODEINFO | AllocBTreeNode |
|
|
* Allocates memory for the node structure as well as the data buffer
|
|
* contained in the structure.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to index parameter block
|
|
*
|
|
* @rdesc Returns a pointer to the newly allocated node
|
|
*************************************************************************/
|
|
|
|
PUBLIC PNODEINFO PASCAL FAR AllocBTreeNode (_LPIPB lpipb)
|
|
{
|
|
PNODEINFO pNode;
|
|
|
|
// Allocate node structure
|
|
if ((pNode = GlobalLockedStructMemAlloc (sizeof (NODEINFO))) == NULL)
|
|
{
|
|
exit0:
|
|
SetErrCode (NULL, E_OUTOFMEMORY);
|
|
return NULL;
|
|
}
|
|
|
|
// Allocate data buffer
|
|
if ((pNode->hMem =
|
|
_GLOBALALLOC (DLLGMEM_ZEROINIT,
|
|
pNode->dwBlockSize = lpipb->BTreeData.Header.dwBlockSize)) == NULL)
|
|
{
|
|
exit1:
|
|
GlobalLockedStructMemFree(pNode);
|
|
goto exit0;
|
|
}
|
|
pNode->pCurPtr = pNode->pBuffer = (LPB)_GLOBALLOCK (pNode->hMem);
|
|
|
|
// Allocate a buffer with the maximum word length, which is the block
|
|
// size
|
|
|
|
if ((pNode->hLastWord =
|
|
_GLOBALALLOC (DLLGMEM_ZEROINIT, pNode->dwBlockSize)) == NULL)
|
|
{
|
|
exit2:
|
|
FreeHandle (pNode->hMem);
|
|
goto exit1;
|
|
}
|
|
pNode->pLastWord = (LPB)_GLOBALLOCK (pNode->hLastWord);
|
|
|
|
// Alllocate temporary result buffer.
|
|
|
|
if ((pNode->hTmp =
|
|
_GLOBALALLOC (DLLGMEM_ZEROINIT, pNode->dwBlockSize)) == NULL)
|
|
{
|
|
FreeHandle (pNode->hLastWord);
|
|
goto exit2;
|
|
}
|
|
pNode->pTmpResult = (LPB)_GLOBALLOCK (pNode->hTmp);
|
|
|
|
return pNode;
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func VOID | FreeBTreeNode |
|
|
* Free all memory allocated for the node
|
|
*
|
|
* @parm PNODEINFO | pNode |
|
|
* BTree node to be freed
|
|
*************************************************************************/
|
|
|
|
PUBLIC VOID PASCAL FAR FreeBTreeNode (PNODEINFO pNode)
|
|
{
|
|
if (pNode == NULL)
|
|
return;
|
|
FreeHandle (pNode->hTmp);
|
|
FreeHandle (pNode->hMem);
|
|
FreeHandle (pNode->hLastWord);
|
|
GlobalLockedStructMemFree(pNode);
|
|
}
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func HRESULT | PrefixCompressWord |
|
|
* Adds a word to a record based on the last word in the node.
|
|
*
|
|
* @parm LPB | pDest |
|
|
* Pointer to the destination buffer
|
|
*
|
|
* @parm LPB | lpbWord |
|
|
* Pointer to the word string to add to node. The format is:
|
|
* - 2-byte: string length
|
|
* - n-byte: the string itself
|
|
* - cbBytePack: real word length
|
|
*
|
|
* @parm LPB | pLastWord |
|
|
* Pointer to the last word entered in the destination buffer
|
|
*
|
|
* @parm int | fOccfLengthSet |
|
|
* Set to 1 if OCCF_LENGTH field is set, else 0
|
|
*
|
|
* @parm PHRESULT | pErrb |
|
|
* Pointer to error structure
|
|
*
|
|
* @rdesc returns number of bytes written to the destination buffer
|
|
* @rcomm
|
|
* Strings are compressed based on how many beginning bytes
|
|
* (prefix) it has in common woth the previous word. The format is
|
|
* - String's length : 2-byte CbPacked
|
|
* - Prefix length : 1-byte (0 - 127). If the high bit is set
|
|
* another word length is to follow the word
|
|
* - Word : n-byte without the prefix
|
|
* - Word's real length - 2-byte CbPacked: only exist if the
|
|
* prefix length high bit is set
|
|
*************************************************************************/
|
|
|
|
PUBLIC int PASCAL FAR PrefixCompressWord
|
|
(LPB pDest, LPB lpbWord, LPB pLastWord, int fOccfLengthSet)
|
|
{
|
|
// Working Variables
|
|
int bPrefix; // The number of prefix bytes that match
|
|
unsigned int wPostfix; // Bytes left over that don't match
|
|
USHORT cbMinWordLen; // Smallest word size between the two words
|
|
LPB pStart = pDest; // Starting position
|
|
DWORD dwRealLength; // The real length of the word
|
|
|
|
|
|
// Get the minimum word length
|
|
wPostfix = GETWORD ((LPUW)lpbWord);
|
|
if ((cbMinWordLen = GETWORD ((LPUW)pLastWord)) > wPostfix)
|
|
cbMinWordLen = (USHORT) wPostfix;
|
|
|
|
// Add one to adjust for two byte word headers (saves an add in the loop)
|
|
cbMinWordLen++;
|
|
|
|
for (bPrefix = 2; bPrefix <= cbMinWordLen; bPrefix++)
|
|
{
|
|
if (lpbWord[bPrefix] != pLastWord[bPrefix])
|
|
break;
|
|
}
|
|
// Adjust back to the real value
|
|
bPrefix -= 2;
|
|
|
|
// Prefix must be <= 127 (high bit is used to indicate fOccfLength field)
|
|
if (bPrefix > 127)
|
|
bPrefix = 127;
|
|
|
|
cbMinWordLen = (USHORT) wPostfix; // Save the word length
|
|
wPostfix -= bPrefix;
|
|
|
|
// Add wLen to wPostfix to get total byte count then write it.
|
|
// The extra byte is for the prefix byte
|
|
pDest += (USHORT)CbBytePack (pDest, (DWORD)(wPostfix + 1));
|
|
|
|
// If WordLen == string length then don't write WordLen
|
|
if (fOccfLengthSet)
|
|
{
|
|
CbByteUnpack (&dwRealLength, lpbWord + sizeof(WORD) + cbMinWordLen );
|
|
if (dwRealLength == cbMinWordLen)
|
|
fOccfLengthSet = FALSE;
|
|
}
|
|
|
|
// Write prefix size
|
|
// If fOccfLengthSet is set, set high bit of bPrefix
|
|
if (fOccfLengthSet)
|
|
*pDest = bPrefix | 0x80;
|
|
else
|
|
*pDest = (BYTE) bPrefix;
|
|
pDest++;
|
|
|
|
// Copy the postfix string over
|
|
MEMCPY (pDest, lpbWord + (bPrefix + sizeof (SHORT)), wPostfix);
|
|
pDest += wPostfix;
|
|
|
|
// if fOccfLengthSet is set append WordLen to end of word
|
|
// (WordLen field follows word in input stream)
|
|
if (fOccfLengthSet)
|
|
pDest += CbBytePack (pDest, dwRealLength);
|
|
|
|
return (int)(pDest - pStart);
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func void | FlushAllNodes |
|
|
* Flushes the remaining nodes to disk when the tree is completely built.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to index block
|
|
*
|
|
* @rdesc S_OK on success or errors if failed
|
|
*
|
|
*************************************************************************/
|
|
|
|
HRESULT PASCAL FlushAllNodes (_LPIPB lpipb)
|
|
{
|
|
PBTREEDATA pTreeData = &lpipb->BTreeData;
|
|
PFILEDATA pOutput = &lpipb->OutFile;
|
|
|
|
PNODEINFO pLeafNode;
|
|
PNODEINFO pStemNode;
|
|
int WordSize;
|
|
BYTE curLevel = 0;
|
|
ERRB errb = S_OK;
|
|
HRESULT fRet;
|
|
|
|
pStemNode = pTreeData->rgpNodeInfo[0];
|
|
|
|
while (pTreeData->rgpNodeInfo[++curLevel] != NULL)
|
|
{
|
|
pLeafNode = pStemNode;
|
|
pStemNode = pTreeData->rgpNodeInfo[curLevel];
|
|
|
|
if ((WordSize = PrefixCompressWord (pStemNode->pCurPtr,
|
|
pLeafNode->pLastWord, pStemNode->pLastWord,
|
|
pTreeData->fOccfLength)) == 0)
|
|
{
|
|
return errb;
|
|
}
|
|
// Save new word as last word
|
|
MEMCPY (pStemNode->pLastWord, pLeafNode->pLastWord,
|
|
GETWORD ((LPUW)pLeafNode->pLastWord) + 2);
|
|
|
|
pStemNode->pCurPtr += WordSize;
|
|
pStemNode->cbLeft -= WordSize;
|
|
#ifdef _DEBUG
|
|
if (pOutput->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
|
|
CopyFileOffset (pStemNode->pCurPtr,
|
|
lpipb->OutFile.foPhysicalOffset);
|
|
pStemNode->pCurPtr += FOFFSET_SIZE;
|
|
pStemNode->cbLeft -= FOFFSET_SIZE;
|
|
#ifdef _DEBUG
|
|
if (pOutput->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
|
|
if (curLevel == 1)
|
|
{
|
|
if ((fRet = WriteLeafNode (lpipb)) != S_OK)
|
|
return(fRet);
|
|
}
|
|
else
|
|
{
|
|
if ((fRet = WriteStemNode (lpipb, pLeafNode)) != S_OK)
|
|
return(fRet);
|
|
}
|
|
}
|
|
// Set the pointer to the top stem node
|
|
pTreeData->Header.foIdxRoot = pOutput->foPhysicalOffset;
|
|
pTreeData->Header.nidIdxRoot = pOutput->foPhysicalOffset.dwOffset;
|
|
|
|
if (curLevel == 1)
|
|
{
|
|
if ((fRet = WriteLeafNode (lpipb)) != S_OK)
|
|
return(fRet);
|
|
}
|
|
else
|
|
{
|
|
if ((fRet = WriteStemNode (lpipb, pStemNode)) != S_OK)
|
|
return(fRet);
|
|
}
|
|
|
|
{
|
|
LONG dwSize;
|
|
|
|
// Flush the output buffer
|
|
if (FileWrite (pOutput->fFile, pOutput->pMem,
|
|
dwSize = (DWORD)(pOutput->pCurrent - pOutput->pMem), &errb) != dwSize)
|
|
return(errb);
|
|
pOutput->foStartOffset = FoAddDw(pOutput->foStartOffset, dwSize);
|
|
}
|
|
|
|
return S_OK;
|
|
}
|
|
|
|
PRIVATE HRESULT PASCAL NEAR WriteBitStreamDWord (PFILEDATA pOutput, DWORD dw,
|
|
int ckeyCenter)
|
|
{
|
|
BYTE ucBits;
|
|
HRESULT fRet;
|
|
|
|
// Bitstream scheme.
|
|
//
|
|
// This writes "dw" one-bits followed by a zero-bit.
|
|
//
|
|
for (; dw;)
|
|
{
|
|
if (dw < cbitBYTE * sizeof(DWORD))
|
|
{
|
|
|
|
ucBits = (BYTE)dw;
|
|
dw = 0;
|
|
}
|
|
else
|
|
{
|
|
ucBits = cbitBYTE * sizeof(DWORD);
|
|
dw -= cbitBYTE * sizeof(DWORD);
|
|
}
|
|
if ((fRet = FWriteBits(pOutput, argdwBits[ucBits],
|
|
(BYTE)ucBits)) != S_OK)
|
|
return fRet;
|
|
}
|
|
return FWriteBool(pOutput, 0);
|
|
}
|
|
|
|
PRIVATE HRESULT PASCAL NEAR WriteFixedDWord (PFILEDATA pOutput, DWORD dw,
|
|
int ckeyCenter)
|
|
{
|
|
// This just writes "ckey.ucCenter" bits of data.
|
|
return (FWriteBits (pOutput, dw, (BYTE)(ckeyCenter + 1)));
|
|
}
|
|
|
|
PRIVATE HRESULT PASCAL NEAR WriteBellDWord (PFILEDATA pOutput, DWORD dw,
|
|
int ckeyCenter)
|
|
{
|
|
BYTE ucBits;
|
|
HRESULT fRet;
|
|
|
|
// The "BELL" scheme is more complicated.
|
|
ucBits = (BYTE)CbitBitsDw(dw);
|
|
|
|
if (ucBits <= ckeyCenter)
|
|
{
|
|
//
|
|
// Encoding a small value. Write a zero, then write
|
|
// "ckey.ucCenter" bits of the value, which
|
|
// is guaranteed to be enough.
|
|
//
|
|
if ((fRet = FWriteBool(pOutput, 0)) != S_OK)
|
|
return fRet;
|
|
return FWriteBits(pOutput, dw, (BYTE)(ckeyCenter));
|
|
}
|
|
|
|
//
|
|
// Encoding a value that won't fit in "ckey.ucCenter" bits.
|
|
// "ucBits" is how many bits it will really take.
|
|
//
|
|
// First, write out "ucBits - ckey.ucCenter" one-bits.
|
|
//
|
|
if ((fRet = FWriteBits(pOutput, argdwBits[ucBits -
|
|
ckeyCenter], (BYTE)(ucBits - ckeyCenter))) != S_OK)
|
|
return fRet;
|
|
//
|
|
// Now, write out the value in "ucBits" bits,
|
|
// but zero the high-bit first.
|
|
//
|
|
return FWriteBits(pOutput, dw & argdwBits[ucBits - 1], ucBits);
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func HRESULT | FWriteBits |
|
|
* Writes a bunch of bits into the output buffer.
|
|
*
|
|
* @parm PFILEDATA | pOutput |
|
|
* Pointer to the output data structure
|
|
*
|
|
* @parm DWORD | dwVal |
|
|
* DWORD value to write
|
|
*
|
|
* @parm BYTE | cbits |
|
|
* Number of bits to write from dwVal
|
|
*
|
|
* @rdesc Returns S_OK on success or errors if failed
|
|
*
|
|
*************************************************************************/
|
|
|
|
PUBLIC HRESULT FAR PASCAL FWriteBits (PFILEDATA pOutput, DWORD dwVal, BYTE cBits)
|
|
{
|
|
BYTE cbitThisPassBits;
|
|
BYTE bThis;
|
|
ERRB errb;
|
|
static DWORD Count = 0;
|
|
|
|
// Loop until no bits left
|
|
for (; cBits;)
|
|
{
|
|
|
|
if (pOutput->ibit < 0)
|
|
{
|
|
pOutput->pCurrent++;
|
|
pOutput->foPhysicalOffset =
|
|
FoAddDw (pOutput->foPhysicalOffset, 1);
|
|
pOutput->cbLeft--;
|
|
#ifdef _DEBUG
|
|
if (pOutput->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
|
|
// Room left in output buffer?
|
|
if (pOutput->cbLeft <= 256)
|
|
{
|
|
LONG dwSize;
|
|
|
|
if (FileWrite (pOutput->fFile, pOutput->pMem,
|
|
dwSize = (DWORD)(pOutput->pCurrent - pOutput->pMem), &errb) !=
|
|
dwSize)
|
|
return(errb);
|
|
|
|
pOutput->cbLeft = pOutput->dwMax;
|
|
pOutput->pCurrent = pOutput->pMem;
|
|
pOutput->foStartOffset = FoAddDw(pOutput->foStartOffset,
|
|
dwSize);
|
|
#ifdef _DEBUG
|
|
// MEMSET (pOutput->pMem, 0, pOutput->dwMax);
|
|
// Count++;
|
|
// if (!FoEquals(pOutput->foStartOffset, pOutput->foPhysicalOffset))
|
|
// _asm int 3;
|
|
#endif
|
|
}
|
|
pOutput->ibit = cbitBYTE - 1;
|
|
}
|
|
else
|
|
{ // Write some bits.
|
|
cbitThisPassBits = (pOutput->ibit + 1 < cBits) ?
|
|
pOutput->ibit + 1 : cBits;
|
|
bThis = (pOutput->ibit == cbitBYTE - 1) ?
|
|
0 : *pOutput->pCurrent;
|
|
bThis |= ((dwVal >> (cBits - cbitThisPassBits)) <<
|
|
(pOutput->ibit - cbitThisPassBits + 1));
|
|
*pOutput->pCurrent = (BYTE)bThis;
|
|
pOutput->ibit -= cbitThisPassBits;
|
|
cBits -= (BYTE)cbitThisPassBits;
|
|
}
|
|
}
|
|
return S_OK;
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc PRIVATE INDEXING
|
|
*
|
|
* @func HRESULT | FWriteBool |
|
|
* Writes a single bit into the output buffer.
|
|
*
|
|
* @parm PFILEDATA | pOutput |
|
|
* Pointer to the output data structure
|
|
*
|
|
* @parm BOOL | dwVal |
|
|
* BOOL value to write
|
|
*
|
|
* @rdesc Returns S_OK on success or errors if failed
|
|
*
|
|
*************************************************************************/
|
|
|
|
PRIVATE HRESULT NEAR PASCAL FWriteBool (PFILEDATA pOutput, BOOL fVal)
|
|
{
|
|
HRESULT fRet = E_FAIL;
|
|
ERRB errb;
|
|
|
|
if (pOutput->ibit < 0)
|
|
{ // This byte is full, point to a new byte
|
|
pOutput->pCurrent++;
|
|
pOutput->foPhysicalOffset =
|
|
FoAddDw (pOutput->foPhysicalOffset, 1);
|
|
pOutput->cbLeft--;
|
|
#ifdef _DEBUG
|
|
if (pOutput->cbLeft <= 0)
|
|
SetErrCode (NULL, E_ASSERT);
|
|
#endif
|
|
|
|
// Room left in output buffer?
|
|
if (pOutput->cbLeft <= sizeof(DWORD))
|
|
{
|
|
LONG dwSize;
|
|
if (FileWrite (pOutput->fFile, pOutput->pMem,
|
|
dwSize = (DWORD)(pOutput->pCurrent - pOutput->pMem), &errb) != dwSize)
|
|
return(errb);
|
|
|
|
pOutput->pCurrent = pOutput->pMem;
|
|
pOutput->cbLeft = pOutput->dwMax;
|
|
pOutput->foStartOffset = FoAddDw(pOutput->foStartOffset, dwSize);
|
|
#ifdef _DEBUG
|
|
MEMSET (pOutput->pMem, 0, pOutput->dwMax);
|
|
#endif
|
|
}
|
|
pOutput->ibit = cbitBYTE - 1;
|
|
}
|
|
if (pOutput->ibit == cbitBYTE - 1) // Zero out a brand-new byte.
|
|
*pOutput->pCurrent = (BYTE)0;
|
|
if (fVal) // Write my boolean.
|
|
*pOutput->pCurrent |= 1 << pOutput->ibit;
|
|
pOutput->ibit--;
|
|
return S_OK; // Fine.
|
|
}
|
|
|
|
|
|
HRESULT PASCAL FAR BuildBtreeFromEso (HFPB hfpb,
|
|
LPSTR pstrFilename, LPB lpbEsiFile,
|
|
LPB lpbEsoFile, PINDEXINFO pIndexInfo)
|
|
{
|
|
_LPIPB lpipb;
|
|
HRESULT fRet;
|
|
ERRB errb;
|
|
BYTE bKeyIndex = 0;
|
|
IPB ipb;
|
|
HFILE hFile;
|
|
|
|
if ((lpipb = MVIndexInitiate(pIndexInfo, NULL)) == NULL)
|
|
return E_OUTOFMEMORY;
|
|
|
|
/* Read in the external sort buffer info */
|
|
|
|
if ((hFile = _lopen (lpbEsiFile, READ)) == HFILE_ERROR)
|
|
return E_NOTEXIST;
|
|
|
|
/* Read old IPB info */
|
|
_lread (hFile, &ipb, sizeof(IPB));
|
|
|
|
/* Transfer meaningful data */
|
|
|
|
lpipb->dwIndexedWord = ipb.dwIndexedWord;
|
|
lpipb->dwUniqueWord = ipb.dwUniqueWord;
|
|
lpipb->dwByteCount = ipb.dwByteCount;
|
|
lpipb->dwOccOffbits = ipb.dwOccOffbits;
|
|
lpipb->dwOccExtbits = ipb.dwOccExtbits;
|
|
lpipb->dwMaxFieldId = ipb.dwMaxFieldId;
|
|
lpipb->dwMaxWCount = ipb.dwMaxWCount;
|
|
lpipb->dwMaxOffset = ipb.dwMaxOffset;
|
|
lpipb->dwTotal3bWordLen = ipb.dwTotal3bWordLen;
|
|
lpipb->dwTotal2bWordLen = ipb.dwTotal2bWordLen;
|
|
lpipb->dwTotalUniqueWordLen = ipb.dwTotalUniqueWordLen;
|
|
lpipb->lcTopics = ipb.lcTopics;
|
|
lpipb->dwMaxTopicId = ipb.dwMaxTopicId;
|
|
// lpipb->dwMemAllowed = ipb.dwMemAllowed;
|
|
lpipb->dwMaxRecordSize = ipb.dwMaxRecordSize;
|
|
lpipb->dwMaxEsbRecSize = ipb.dwMaxEsbRecSize;
|
|
lpipb->dwMaxWLen = ipb.dwMaxWLen;
|
|
lpipb->idxf = ipb.idxf;
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
if ((lpipb->wi.hSigma = _GLOBALALLOC (DLLGMEM_ZEROINIT,
|
|
(LCB)((lpipb->dwMaxTopicId + 1) * sizeof (SIGMA)))) == NULL)
|
|
return SetErrCode (&errb, E_OUTOFMEMORY);
|
|
lpipb->wi.hrgsigma = (HRGSIGMA)_GLOBALLOCK (lpipb->wi.hSigma);
|
|
|
|
if ((lpipb->wi.hLog = _GLOBALALLOC (DLLGMEM_ZEROINIT,
|
|
(CB)(cLOG_MAX * sizeof (FLOAT)))) == NULL)
|
|
{
|
|
SetErrCode (&errb, (HRESULT)(fRet = E_OUTOFMEMORY));
|
|
exit1:
|
|
FreeHandle (lpipb->wi.hSigma);
|
|
MVIndexDispose (lpipb);
|
|
return fRet;
|
|
}
|
|
#if 0
|
|
lpipb->wi.lrgrLog = (FLOAT FAR *)_GLOBALLOCK (lpipb->wi.hLog);
|
|
// Initialize the array
|
|
for (loop = cLOG_MAX - 1; loop > 0; --loop)
|
|
{
|
|
rLog = (FLOAT)1.0 / (float)loop;
|
|
lpipb->wi.lrgrLog[loop] = rLog * rLog;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
|
|
|
|
// Build the permanent index
|
|
fRet = BuildBTree(NULL, lpipb, lpbEsoFile, hfpb, pstrFilename);
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
FreeHandle (lpipb->wi.hLog);
|
|
goto exit1;
|
|
}
|
|
fRet = S_OK;
|
|
goto exit1;
|
|
}
|