2059 lines
66 KiB
C
2059 lines
66 KiB
C
#define VER3
|
|
/*************************************************************************
|
|
* *
|
|
* INDEX.C *
|
|
* *
|
|
* Copyright (C) Microsoft Corporation 1990-1994 *
|
|
* All Rights reserved. *
|
|
* *
|
|
**************************************************************************
|
|
* *
|
|
* Module Intent *
|
|
* This is the second stage of the index building process. After all *
|
|
* of the word have been add in stage 1, IndexBuild will be called. *
|
|
* IndexBuild starts the second stage. We will merge-sort the temp file *
|
|
* generated in phase 1 to create a second temp file to send to phase 3. *
|
|
* *
|
|
**************************************************************************
|
|
* *
|
|
* Current Owner: BinhN *
|
|
* *
|
|
**************************************************************************/
|
|
|
|
#include <mvopsys.h>
|
|
#include <mem.h>
|
|
#include <memory.h>
|
|
#include <io.h>
|
|
#include <math.h>
|
|
#include <mvsearch.h>
|
|
#include <orkin.h>
|
|
#include "common.h"
|
|
#include "index.h"
|
|
|
|
#ifdef _DEBUG
|
|
static BYTE NEAR s_aszModule[] = __FILE__; /* Used by error return functions.*/
|
|
#endif
|
|
|
|
|
|
#ifndef _32BIT
|
|
#define ESOUTPUT_BUFFER 0xFFFC // Size of output file buffer
|
|
// This must be at the size of the largest word + 12
|
|
// or word + 14 if OCCF_LENGTH is set
|
|
#else
|
|
#define ESOUTPUT_BUFFER 0xFFFFC // Size of output file buffer
|
|
// This must be at the size of the largest word + 12
|
|
// or word + 14 if OCCF_LENGTH is set
|
|
#endif
|
|
|
|
#define FLUSH_NEW_RECORD 1
|
|
#define FLUSH_EXCEPT_LAST 2
|
|
|
|
/*************************************************************************
|
|
*
|
|
* INTERNAL PRIVATE FUNCTIONS
|
|
*
|
|
* All of them should be declared near
|
|
*
|
|
*************************************************************************/
|
|
PRIVATE HRESULT NEAR PASCAL FillInputBuffer (LPESB, HFPB);
|
|
PRIVATE HRESULT NEAR PASCAL ESFlushBuffer (LPESI);
|
|
PRIVATE HRESULT NEAR PASCAL ESFillBuffer (_LPIPB, LPESB);
|
|
PRIVATE HRESULT NEAR PASCAL ESMemory2Disk (_LPIPB, PMERGEHEADER, int);
|
|
PRIVATE HRESULT NEAR PASCAL ProcessFiles (_LPIPB lpipb, LPMERGEPARAMS);
|
|
PRIVATE int NEAR PASCAL CompareRecordBuffers (_LPIPB, LPB, LPB);
|
|
PRIVATE VOID NEAR PASCAL PQueueUp (_LPIPB, LPESB FAR *, LONG);
|
|
PRIVATE VOID NEAR PASCAL PQueueDown (_LPIPB);
|
|
PRIVATE PTOPICDATA PASCAL NEAR MergeTopicNode (PMERGEHEADER, PTOPICDATA, int);
|
|
PRIVATE VOID NEAR MergeOccurrence (PTOPICDATA, PTOPICDATA, int);
|
|
PRIVATE LPV PASCAL NEAR GetBlockNode (PBLKCOMBO lpBlockCombo);
|
|
PRIVATE VOID PASCAL NEAR SetQueue (LPESI pEsi);
|
|
PRIVATE HRESULT PASCAL NEAR ESBBlockAllocate (_LPIPB lpipb, DWORD lMemSize);
|
|
PRIVATE BOOL PASCAL LoadEsiTemp (_LPIPB lpipb, LPESI lpesi, LPB lpbEsiFile,
|
|
LPB lpbIsiFile, PHRESULT phr);
|
|
PRIVATE VOID PASCAL NEAR SaveEsiTemp (LPIPB lpipb, LPESI lpesi);
|
|
PRIVATE VOID PASCAL NEAR UpdateEsiTemp (LPIPB lpipb);
|
|
PRIVATE BOOL PASCAL NEAR FindTopic(LPMERGEPARAMS lpmp, DWORD dwTopicId);
|
|
|
|
/*************************************************************************
|
|
*
|
|
* INTERNAL PUBLIC FUNCTIONS
|
|
*
|
|
* All of them should be declared far, unless we know they belong to
|
|
* the same segment. They should be included in some include file
|
|
*
|
|
*************************************************************************/
|
|
PUBLIC HRESULT FAR PASCAL FlushTree(_LPIPB lpipb);
|
|
PUBLIC HRESULT FAR PASCAL MergeSortTreeFile (_LPIPB, LPMERGEPARAMS);
|
|
HRESULT FAR PASCAL AllocSigmaTable (_LPIPB lpipb);
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc EXTERNAL API INDEX
|
|
*
|
|
* @func BOOL FAR PASCAL | MVIndexBuild |
|
|
* This function will build an index file based on the information
|
|
* collected in the Index parameter block.
|
|
*
|
|
* @parm HFPB | hSysFile |
|
|
* If it is non-null, it is the handle of an already opened system file.
|
|
* In this case the index is a subfile of the opened system file
|
|
* If it is 0, the index file is a regular DOS file
|
|
*
|
|
* @parm LPIPB | lpipb |
|
|
* Pointer to Index Parameter Block. This structure contains all the
|
|
* information necessary to build the index file
|
|
*
|
|
* @parm HFPB | hfpb |
|
|
* Index hfpb if pstrFile is NULL
|
|
*
|
|
* @parm LPSTR | pstrFile |
|
|
* Index filename if hfpb is NULL
|
|
*
|
|
* @rdesc S_OK, or other errors
|
|
*
|
|
* @xref MVIndexInitiate()
|
|
*************************************************************************/
|
|
/*
|
|
* This operates in three main steps:
|
|
*
|
|
* 1. Send finish to first phase to dump the buffer. Then merge-sort
|
|
* that file into a temporary index. Keep statistics on the information
|
|
* written to this temporary index.
|
|
*
|
|
* 2. Analyze the statistics gathered during the temporary index
|
|
* building phase. This analysis results in the choice of
|
|
* compression processes that will be used in the next step.
|
|
*
|
|
* 3. Permanent index building phase. During this phase, the
|
|
* temporary index is read, compressed like crazy, and written
|
|
* to a permanent index file. Unlike the temporary index, the
|
|
* permanent index contains directory nodes as well as leaf
|
|
* nodes.
|
|
*
|
|
*************************************************************************/
|
|
|
|
PUBLIC HRESULT EXPORT_API FAR PASCAL MVIndexBuild (HFPB hSysFile,
|
|
_LPIPB lpipb, HFPB hfpb, LPSTR pstrFile)
|
|
{
|
|
ERRB errb;
|
|
PHRESULT phr = &errb;
|
|
BYTE bKeyIndex = CKEY_OCC_BASE; // Index into cKey array for compression
|
|
HRESULT fRet; // Return value from this function.
|
|
DWORD loop;
|
|
|
|
// Sanity check
|
|
if (lpipb == NULL || (NULL == hfpb && NULL == pstrFile))
|
|
return E_INVALIDARG;
|
|
|
|
// Flush the internal sort
|
|
// Flushes any records in the tree to disk
|
|
fRet = FlushTree(lpipb);
|
|
|
|
// Free all memory blocks
|
|
FreeISI (lpipb);
|
|
|
|
if (fRet != S_OK)
|
|
return(fRet);
|
|
|
|
// lpipb->lcTopics++; // Adjust to base-1 from base-0
|
|
|
|
if (lpipb->esi.cesb == 0)
|
|
// Nothing to process, there will be no index file
|
|
return S_OK;
|
|
|
|
if (lpipb->idxf & KEEP_TEMP_FILE)
|
|
SaveEsiTemp (lpipb, &lpipb->esi);
|
|
|
|
// If we're doing term-weighting, set up a huge array to contain the
|
|
// sigma terms. The size of the array depends on the total # of topics
|
|
// We also create an array of LOG values to save calculations later
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
if ((fRet = AllocSigmaTable (lpipb)) != S_OK)
|
|
return(fRet);
|
|
}
|
|
|
|
if ((fRet = MergeSortTreeFile (lpipb, NULL)) != S_OK)
|
|
return SetErrCode (phr, fRet);
|
|
if ((lpipb->idxf & KEEP_TEMP_FILE) == 0)
|
|
FileUnlink (NULL, lpipb->isi.aszTempName, REGULAR_FILE);
|
|
|
|
// If we are doing term-weighting we have to square root all sigma values
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
// ISBU_IR_CHANGE not needed here 'cos computing sqrt is necessary in both cases
|
|
for (loop = 0; loop < lpipb->dwMaxTopicId + 1; ++loop)
|
|
lpipb->wi.hrgsigma[loop] =
|
|
(float)sqrt ((double)lpipb->wi.hrgsigma[loop]);
|
|
}
|
|
|
|
// Analyze data to get the best compression scheme
|
|
// TopicId
|
|
|
|
// Note: We can't use fixed field compression for topic, since they
|
|
// can be modified by update. A fixed field format may become
|
|
// insufficient to store larger values of topic differences
|
|
|
|
VGetBestScheme(&lpipb->cKey[CKEY_TOPIC_ID],
|
|
&lpipb->BitCount[CKEY_TOPIC_ID][0], lcbitBITSTREAM_ILLEGAL, TRUE);
|
|
|
|
// Occurrence Count
|
|
VGetBestScheme(&lpipb->cKey[CKEY_OCC_COUNT],
|
|
&lpipb->BitCount[CKEY_OCC_COUNT][0], lcbitBITSTREAM_ILLEGAL, TRUE);
|
|
|
|
if (lpipb->occf & OCCF_COUNT)
|
|
{
|
|
VGetBestScheme(&lpipb->cKey[bKeyIndex],
|
|
&lpipb->BitCount[bKeyIndex][0], lcbitBITSTREAM_ILLEGAL, TRUE);
|
|
bKeyIndex++;
|
|
}
|
|
|
|
if (lpipb->occf & OCCF_OFFSET)
|
|
{
|
|
VGetBestScheme(&lpipb->cKey[bKeyIndex],
|
|
&lpipb->BitCount[bKeyIndex][0], lcbitBITSTREAM_ILLEGAL, TRUE);
|
|
bKeyIndex++;
|
|
}
|
|
|
|
if (lpipb->idxf & KEEP_TEMP_FILE)
|
|
UpdateEsiTemp (lpipb);
|
|
|
|
// Build the permanent index
|
|
fRet = BuildBTree(hSysFile, lpipb, lpipb->esi.aszTempName, hfpb, pstrFile);
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
FreeHandle (lpipb->wi.hSigma);
|
|
FreeHandle (lpipb->wi.hLog);
|
|
}
|
|
return fRet;
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc INDEX
|
|
*
|
|
* @func HRESULT NEAR PASCAL | FillInputBuffer |
|
|
* Fills the buffer by reading from the specified file.
|
|
*
|
|
* @parm PESB | pEsb |
|
|
* Pointer to external sort block to fill
|
|
*
|
|
* @parm HFPB | hFile |
|
|
* Handle to the input file
|
|
*
|
|
* @rdesc S_OK, or errors if failed
|
|
*
|
|
*************************************************************************/
|
|
|
|
HRESULT NEAR PASCAL FillInputBuffer(LPESB pEsb, HFPB hFile)
|
|
{
|
|
ERRB errb;
|
|
DWORD dwBytesRead;
|
|
|
|
// Read in data
|
|
if ((dwBytesRead = FileSeekRead (hFile,
|
|
(LPB)pEsb->lrgbMem, pEsb->lfo, pEsb->dwEsbSize, &errb)) == 0)
|
|
return errb;
|
|
|
|
// Update utility variables
|
|
pEsb->lfo = FoAddDw(pEsb->lfo, dwBytesRead);
|
|
pEsb->dwEsbSize = (CB)dwBytesRead;
|
|
pEsb->ibBuf = 0;
|
|
return S_OK;
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc INDEX
|
|
*
|
|
* @func HRESULT NEAR PASCAL | ESFlushBuffer |
|
|
* Flushes the output buffer to disk and resets it.
|
|
*
|
|
* @parm LPESI | pEsi |
|
|
* Pointer to ESI block
|
|
*
|
|
* @rdesc S_OK, or errors if failed
|
|
*
|
|
*************************************************************************/
|
|
|
|
HRESULT NEAR PASCAL ESFlushBuffer(LPESI pEsi)
|
|
{
|
|
ERRB errb;
|
|
DWORD dwLen;
|
|
|
|
dwLen = pEsi->ibBuf;
|
|
if (dwLen != (DWORD)FileWrite (pEsi->hfpb, pEsi->pOutputBuffer,
|
|
dwLen, &errb))
|
|
return errb;
|
|
|
|
pEsi->lfoTempOffset = FoAddDw (pEsi->lfoTempOffset, dwLen);
|
|
pEsi->ibBuf = 0;
|
|
return S_OK;
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc INDEX
|
|
*
|
|
* @func HRESULT NEAR PASCAL | ESFillBuffer |
|
|
* Updates the input buffer with new data from the input file.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to index parameter block
|
|
*
|
|
* @parm LPESB | pEsb |
|
|
* Pointer to ESB block to be filled
|
|
*
|
|
* @rdesc S_OK, or other errors
|
|
*************************************************************************/
|
|
|
|
HRESULT NEAR PASCAL ESFillBuffer(_LPIPB lpipb, LPESB pEsb)
|
|
{
|
|
DWORD dwBytesRead;
|
|
DWORD dwExtra = pEsb->dwEsbSize - pEsb->ibBuf;
|
|
ERRB errb;
|
|
|
|
// Read either the entire buffer size or whatever is left
|
|
dwBytesRead = DwSubFo (pEsb->lfoMax, pEsb->lfo);
|
|
|
|
if (dwBytesRead > pEsb->dwEsbSize - dwExtra)
|
|
dwBytesRead = pEsb->dwEsbSize - dwExtra;
|
|
|
|
// Save unproccessed information to beginning of buffer
|
|
if (dwExtra)
|
|
MEMMOVE ((LPB)pEsb->lrgbMem, pEsb->lrgbMem + pEsb->ibBuf, dwExtra);
|
|
|
|
// Read in the new data
|
|
if ((dwBytesRead = FileSeekRead (lpipb->isi.hfpb, (LPB)(pEsb->lrgbMem +
|
|
dwExtra), pEsb->lfo, dwBytesRead, &errb)) == 0 &&
|
|
errb != S_OK)
|
|
return(errb);
|
|
|
|
pEsb->lfo = FoAddDw(pEsb->lfo, dwBytesRead);
|
|
pEsb->ibBuf = 0;
|
|
pEsb->dwEsbSize = dwBytesRead + dwExtra;
|
|
return(S_OK);
|
|
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc INTERNAL INDEXING
|
|
*
|
|
* @func HRESULT FAR PASCAL | MergeSortTree File |
|
|
* Sorts the file generated from the tree output into one
|
|
* list of sorted elements.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to index parameter block
|
|
*
|
|
*************************************************************************/
|
|
|
|
PUBLIC HRESULT PASCAL FAR MergeSortTreeFile (_LPIPB lpipb, LPMERGEPARAMS lpmp)
|
|
{
|
|
// Local replacement variables
|
|
LPESI pEsi; // Pointer to external sort info
|
|
LPISI pIsi; // Pointer to internal sort info
|
|
HFPB hInputFile; // Handle to input file
|
|
ERRB errb;
|
|
PHRESULT phr = &errb;
|
|
DWORD cesb; // Input buffer count
|
|
LPESB FAR* lrgPriorityQueue; // Pointer to Priority Queue
|
|
WORD uiQueueSize = 0; // Count of entries in Queue
|
|
DWORD dwBufferSize;
|
|
|
|
// Working variables
|
|
HRESULT fRet;
|
|
LPESB pEsb; // Temp pointer to linked list
|
|
|
|
// Sanity check
|
|
if (lpipb == NULL)
|
|
return E_INVALIDARG;
|
|
|
|
// Variables initialization
|
|
pEsi = &lpipb->esi; // Pointer to external sort info
|
|
pIsi = &lpipb->isi; // Pointer to internal sort info
|
|
cesb = pEsi->cesb; // Input buffer count
|
|
|
|
// Open input file
|
|
if ((pIsi->hfpb = FileOpen (NULL, pIsi->aszTempName,
|
|
REGULAR_FILE, READ, phr)) == NULL)
|
|
return *phr;
|
|
|
|
hInputFile = pIsi->hfpb;
|
|
|
|
// Allocate & fill input buffers
|
|
for (pEsb = pEsi->lpesbRoot; pEsb != NULL; pEsb = pEsb->lpesbNext)
|
|
{
|
|
DWORD cbRead;
|
|
|
|
dwBufferSize = (lpipb->dwMemAllowed * 6) / (8 * pEsi->cesb);
|
|
|
|
// Alocate buffer space
|
|
if ((pEsb->hMem = _GLOBALALLOC (DLLGMEM_ZEROINIT,
|
|
dwBufferSize)) == NULL)
|
|
{
|
|
fRet = E_OUTOFMEMORY;
|
|
exit1:
|
|
FreeEsi (lpipb);
|
|
FileClose(hInputFile);
|
|
pIsi->hfpb = NULL;
|
|
return fRet;
|
|
}
|
|
pEsb->lrgbMem = (LRGB)_GLOBALLOCK (pEsb->hMem);
|
|
|
|
if ((cbRead = DwSubFo(pEsb->lfoMax, pEsb->lfo)) > dwBufferSize)
|
|
cbRead = dwBufferSize;
|
|
|
|
// Fill buffer from disk
|
|
if (FileSeekRead (hInputFile, pEsb->lrgbMem, pEsb->lfo,
|
|
cbRead, phr) != (LONG)cbRead)
|
|
{
|
|
fRet = *phr;
|
|
_GLOBALUNLOCK(pEsb->hMem);
|
|
_GLOBALFREE(pEsb->hMem);
|
|
pEsb->hMem = NULL;
|
|
goto exit1;
|
|
}
|
|
|
|
pEsb->dwEsbSize = cbRead;
|
|
pEsb->ibBuf = 0;
|
|
pEsb->lfo = FoAddDw (pEsb->lfo, cbRead);
|
|
}
|
|
|
|
// Allocate a priority queue array. The size of the array
|
|
// is the number of external sort info blocks plus 1, since
|
|
// location 0 is not used.
|
|
if ((pEsi->hPriorityQueue = _GLOBALALLOC (DLLGMEM_ZEROINIT,
|
|
(DWORD)(pEsi->cesb + 1) * sizeof (LPB))) == NULL)
|
|
{
|
|
fRet = E_OUTOFMEMORY;
|
|
goto exit1;
|
|
}
|
|
pEsi->lrgPriorityQueue =
|
|
(LPESB FAR *)_GLOBALLOCK (pEsi->hPriorityQueue);
|
|
lrgPriorityQueue = pEsi->lrgPriorityQueue;
|
|
|
|
// Attach input buffers to Priority Queue
|
|
// Remebering to start at offset 1 NOT 0 (PQ's have a null 0 element)
|
|
for (pEsb = pEsi->lpesbRoot; pEsb != NULL; pEsb = pEsb->lpesbNext)
|
|
{
|
|
lrgPriorityQueue[++uiQueueSize] = pEsb;
|
|
PQueueUp (lpipb, lrgPriorityQueue, uiQueueSize);
|
|
}
|
|
pEsi->uiQueueSize = uiQueueSize;
|
|
|
|
// Clear largest Record Size field
|
|
// lpipb->dwMaxRecordSize = 0;
|
|
|
|
fRet = ProcessFiles(lpipb, lpmp);
|
|
_GLOBALUNLOCK (pEsi->hPriorityQueue);
|
|
_GLOBALFREE (pEsi->hPriorityQueue);
|
|
pEsi->hPriorityQueue = NULL;
|
|
goto exit1;
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc INDEX
|
|
*
|
|
* @func HRESULT NEAR PASCAL | ESMemory2Disk |
|
|
* Copies temp record to output buffer.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to index parameter block
|
|
*
|
|
* @parm PMERGEHEADER | pHeader |
|
|
* Pointer to header to flush
|
|
*
|
|
* @parm int | flag |
|
|
* - if FLUSH_NEW_RECORD, the flush is due to new record, we flush
|
|
* everything, else we may do a partial flush only
|
|
* - if FLUSH_EXCEPT_LAST, we don't flush the last topic
|
|
*
|
|
* @rdesc S_OK, or other errors
|
|
*************************************************************************/
|
|
PRIVATE HRESULT NEAR PASCAL ESMemory2Disk
|
|
(_LPIPB lpipb, PMERGEHEADER pHeader, int flag)
|
|
{
|
|
// Local replacement variables
|
|
LPESI pEsi = &lpipb->esi;
|
|
LPB pMax = pEsi->pOutputBuffer + ESOUTPUT_BUFFER - 2 * sizeof(DWORD);
|
|
DWORD dwOccCount;
|
|
LPB pOutputBuffer = pEsi->pOutputBuffer;
|
|
ERRB errb;
|
|
PHRESULT phr = &errb;
|
|
HRESULT fRet;
|
|
BYTE cNumOcc;
|
|
OCCF occf;
|
|
|
|
// Working variables
|
|
PTOPICDATA pTopic; // Temp var to traverse the topic linked list
|
|
DWORD loop, sub; // Various loop counters
|
|
DWORD dwTopicIdDelta;
|
|
DWORD OccDelta[5]; // Delta base for all occurrence data
|
|
DWORD LastOcc[5];
|
|
FLOAT rLog; // (1/n) - IDXF_NORMALIZE is set
|
|
FLOAT rLogSquared; // (1/n)^2 - IDXF_NORMALIZE is set
|
|
LPB pStart;
|
|
LPB pCurPtr;
|
|
|
|
// Set up pointers
|
|
pStart = pCurPtr = pOutputBuffer + pEsi->ibBuf;
|
|
|
|
// Variable replacement
|
|
occf = lpipb->occf;
|
|
|
|
// Size of string
|
|
loop = pHeader->dwStrLen;
|
|
|
|
// Make sure the string, FileId, Topic Count and Record Size fit
|
|
// We add in and extra DWORD for 5 byte compresssion problems and
|
|
// to cover the Word Length if there is one.
|
|
if ((pStart + loop + sizeof (DWORD) * 5) >= pMax)
|
|
{
|
|
if ((fRet = ESFlushBuffer (pEsi)) != S_OK)
|
|
return(fRet);
|
|
|
|
pStart = pCurPtr = pOutputBuffer;
|
|
}
|
|
|
|
if (pHeader->fEmitRecord == FALSE)
|
|
{
|
|
// If we never emitted the record header then we emitted now
|
|
// Reset the flag
|
|
pHeader->fEmitRecord = TRUE;
|
|
|
|
// Skip record size field
|
|
pCurPtr += sizeof (DWORD);
|
|
|
|
// Pascal string
|
|
MEMCPY (pCurPtr, pHeader->lpbWord, loop);
|
|
pCurPtr += loop;
|
|
|
|
// Word Length
|
|
if (occf & OCCF_LENGTH)
|
|
pCurPtr += CbBytePack (pCurPtr, pHeader->dwWordLength);
|
|
|
|
// FieldId
|
|
if (occf & OCCF_FIELDID)
|
|
pCurPtr += CbBytePack (pCurPtr, pHeader->dwFieldId);
|
|
|
|
// Topic Count
|
|
if (flag & FLUSH_NEW_RECORD)
|
|
{
|
|
// This is the whole record. dwTopicCount value is correct
|
|
SETLONG((LPUL)pCurPtr, pHeader->dwTopicCount);
|
|
}
|
|
else
|
|
{
|
|
// Save the offset for backpatching
|
|
pHeader->foTopicCount = FoAddDw (pEsi->lfoTempOffset,
|
|
(DWORD)(pCurPtr - pOutputBuffer));
|
|
pHeader->pTopicCount = pCurPtr;
|
|
}
|
|
pCurPtr += sizeof(DWORD);
|
|
|
|
// Write Record Length
|
|
*(LPUL)pStart = (DWORD)(pCurPtr - pStart - sizeof (DWORD));
|
|
|
|
}
|
|
else if (flag & FLUSH_NEW_RECORD)
|
|
{
|
|
// We emit the record before, since pheader->fEmitRecord == TRUE
|
|
// We need to backpatch the topic count
|
|
if (FoCompare(pHeader->foTopicCount, pEsi->lfoTempOffset) >= 0)
|
|
{
|
|
// Everything is still in memory, just do local backpatch
|
|
SETLONG((LPUL)(pHeader->pTopicCount), pHeader->dwTopicCount);
|
|
}
|
|
else
|
|
{
|
|
// Do backpatch in the file by seeking back to the right
|
|
// place
|
|
if (FileSeekWrite(pEsi->hfpb, &pHeader->dwTopicCount,
|
|
pHeader->foTopicCount, sizeof(DWORD), phr) != sizeof(DWORD))
|
|
return(*phr);
|
|
|
|
// Restore the current file offset
|
|
FileSeek(pEsi->hfpb, pEsi->lfoTempOffset, 0, phr);
|
|
}
|
|
}
|
|
|
|
// Convert all occ data to delta values & compress them
|
|
pTopic = pHeader->pTopic;
|
|
cNumOcc = lpipb->ucNumOccDataFields;
|
|
|
|
for (; pTopic;)
|
|
{
|
|
POCCDATA pOccData;
|
|
PTOPICDATA pReleased;
|
|
|
|
if ((flag & FLUSH_EXCEPT_LAST) && pTopic->pNext == NULL)
|
|
break;
|
|
|
|
// Set TopicId delta
|
|
dwTopicIdDelta = pTopic->dwTopicId - pHeader->dwLastTopicId;
|
|
pHeader->dwLastTopicId = pTopic->dwTopicId;
|
|
|
|
// Save bit size to the statistics array
|
|
lpipb->BitCount[CKEY_TOPIC_ID][CbitBitsDw (dwTopicIdDelta)] += 1;
|
|
|
|
// Write TopicID Delta
|
|
if (pCurPtr > pMax)
|
|
{
|
|
pEsi->ibBuf = (DWORD)(pCurPtr - pOutputBuffer);
|
|
if ((fRet = ESFlushBuffer (pEsi)) != S_OK)
|
|
return(fRet);
|
|
pCurPtr = pOutputBuffer;
|
|
}
|
|
pCurPtr += CbBytePack (pCurPtr, dwTopicIdDelta);
|
|
|
|
if (cNumOcc == 0)
|
|
{
|
|
pReleased = pTopic;
|
|
pTopic = pTopic->pNext;
|
|
|
|
// Add the released to the freed linked list
|
|
pReleased->pNext = (PTOPICDATA)lpipb->TopicBlock.pFreeList;
|
|
lpipb->TopicBlock.pFreeList = (PLIST)pReleased;
|
|
lpipb->TopicBlock.dwCount--;
|
|
continue;
|
|
}
|
|
|
|
if (dwOccCount = pTopic->dwOccCount)
|
|
{
|
|
|
|
// Reset count occdata delta for every new topic
|
|
MEMSET (OccDelta, 0, 5 * sizeof (DWORD));
|
|
MEMSET (LastOcc, 0, 5 * sizeof (DWORD));
|
|
|
|
// Copy Occurrence Count
|
|
if (pCurPtr > pMax)
|
|
{
|
|
pEsi->ibBuf = (DWORD)(pCurPtr - pOutputBuffer);
|
|
if ((fRet = ESFlushBuffer (pEsi)) != S_OK)
|
|
return(fRet);
|
|
pCurPtr = pOutputBuffer;
|
|
}
|
|
pCurPtr += CbBytePack (pCurPtr, dwOccCount);
|
|
|
|
// Save bit size to the statistics array
|
|
lpipb->BitCount[1][CbitBitsDw (dwOccCount)] += 1;
|
|
|
|
// Repeat for each occurrence block
|
|
for (pOccData = pTopic->pOccData,
|
|
sub = dwOccCount; sub > 0 && pOccData; --sub)
|
|
{
|
|
LPDW lpDw;
|
|
int iIndex;
|
|
POCCDATA pReleased;
|
|
|
|
if (pCurPtr + 5 * sizeof(DWORD) > pMax)
|
|
{
|
|
pEsi->ibBuf = (DWORD)(pCurPtr - pOutputBuffer);
|
|
if ((fRet = ESFlushBuffer (pEsi)) != S_OK)
|
|
return(fRet);
|
|
pStart = pCurPtr = pOutputBuffer;
|
|
}
|
|
|
|
lpDw = &pOccData->OccData[0];
|
|
iIndex = CKEY_OCC_BASE;
|
|
|
|
if (occf & OCCF_COUNT)
|
|
{
|
|
// Convert each value to a delta value
|
|
OccDelta[iIndex] = *lpDw - LastOcc[iIndex];
|
|
LastOcc[iIndex] = *lpDw;
|
|
lpDw++;
|
|
|
|
// Save to bit size to the statistics array
|
|
lpipb->BitCount[iIndex][CbitBitsDw (OccDelta[iIndex])] += 1;
|
|
|
|
// Compress occurrence field to buffer
|
|
pCurPtr += CbBytePack (pCurPtr, OccDelta[iIndex]);
|
|
iIndex++;
|
|
}
|
|
|
|
if (occf & OCCF_OFFSET)
|
|
{
|
|
// Convert each value to a delta value
|
|
OccDelta[iIndex] = *lpDw - LastOcc[iIndex];
|
|
LastOcc[iIndex] = *lpDw;
|
|
lpDw++;
|
|
|
|
// Save to bit size to the statistics array
|
|
lpipb->BitCount[iIndex][CbitBitsDw (OccDelta[iIndex])] += 1;
|
|
|
|
// Compress occurrence field to buffer
|
|
pCurPtr += CbBytePack (pCurPtr, OccDelta[iIndex]);
|
|
iIndex++;
|
|
}
|
|
|
|
pReleased = pOccData;
|
|
pOccData = pOccData->pNext;
|
|
pReleased->pNext = (POCCDATA)lpipb->OccBlock.pFreeList;
|
|
lpipb->OccBlock.pFreeList = (PLIST)pReleased;
|
|
lpipb->OccBlock.dwCount--;
|
|
}
|
|
|
|
// Check for mismatch between count and links
|
|
#ifdef _DEBUG
|
|
if (sub)
|
|
SetErrCode (phr, E_ASSERT);
|
|
|
|
if (pOccData)
|
|
SetErrCode (phr, E_ASSERT);
|
|
#endif
|
|
}
|
|
|
|
// Update the sigma values if we are doing term weighing
|
|
// erinfox: remove test against flag. Sometimes sigma never
|
|
// got calculated for a topic and that caused a divide by zero
|
|
// later on.
|
|
if ((lpipb->idxf & IDXF_NORMALIZE) /* && (flag & FLUSH_NEW_RECORD)*/)
|
|
{
|
|
|
|
if (pTopic->dwTopicId > lpipb->dwMaxTopicId)
|
|
{
|
|
// Incease the size of the sigma table. This can happen when
|
|
// updating with new topics
|
|
_GLOBALUNLOCK (lpipb->wi.hSigma);
|
|
if ((lpipb->wi.hSigma = _GLOBALREALLOC (lpipb->wi.hSigma,
|
|
(pTopic->dwTopicId + 1) * sizeof(float),
|
|
DLLGMEM_ZEROINIT)) == NULL)
|
|
{
|
|
return (SetErrCode(phr, E_OUTOFMEMORY));
|
|
}
|
|
lpipb->wi.hrgsigma = (HRGSIGMA)_GLOBALLOCK(lpipb->wi.hSigma);
|
|
lpipb->dwMaxTopicId = pTopic->dwTopicId ;
|
|
}
|
|
|
|
if (lpipb->bState == INDEXING_STATE)
|
|
{
|
|
#ifndef ISBU_IR_CHANGE
|
|
FLOAT fOcc;
|
|
|
|
if (pHeader->dwTopicCount >= cLOG_MAX)
|
|
{
|
|
// we have to guard against the possibility of the log resulting in
|
|
// a value <= 0.0. Very rare, but possible in the future. This happens
|
|
// if dwTopicCount approaches or exceeds the N we are using (N == 100 million)
|
|
if (pHeader->dwTopicCount >= cNintyFiveMillion)
|
|
rLog = cVerySmallWt; // log10(100 mil/ 95 mil) == 0.02
|
|
else
|
|
//rLog = (float) log10(cHundredMillion/(double)pHeader->dwTopicCount);
|
|
rLog = (float) (8.0 - log10((double)pHeader->dwTopicCount));
|
|
|
|
rLogSquared = rLog*rLog;
|
|
}
|
|
else
|
|
rLogSquared = lpipb->wi.lrgrLog[(WORD)pHeader->dwTopicCount];
|
|
|
|
// Update sigma value
|
|
// NOTE : We are bounding dwOccCount by a value of eTFThreshold
|
|
// The RHS of the equation below has an upperbound of 2 power 30.
|
|
fOcc = (float) min(cTFThreshold, dwOccCount);
|
|
lpipb->wi.hrgsigma[pTopic->dwTopicId] += (SIGMA) fOcc*fOcc*rLogSquared;
|
|
//(SIGMA) (fOcc * fOcc * rLogSquared/(float)0xFFFF);
|
|
#else
|
|
// Failed for update : UNDONE
|
|
if (pHeader->dwTopicCount >= cLOG_MAX)
|
|
{
|
|
rLog = (float)1.0 / (float)pHeader->dwTopicCount;
|
|
rLogSquared = rLog * rLog;
|
|
}
|
|
else
|
|
rLogSquared = lpipb->wi.lrgrLog[(WORD)pHeader->dwTopicCount];
|
|
// Update sigma value
|
|
lpipb->wi.hrgsigma[pTopic->dwTopicId] +=
|
|
(SIGMA)(dwOccCount * dwOccCount) * rLogSquared;
|
|
#endif // ISBU_IR_CHANGE
|
|
}
|
|
}
|
|
pReleased = pTopic;
|
|
pTopic = pTopic->pNext;
|
|
|
|
// Add the released to the freed linked list
|
|
pReleased->pNext = (PTOPICDATA)lpipb->TopicBlock.pFreeList;
|
|
lpipb->TopicBlock.pFreeList = (PLIST)pReleased;
|
|
lpipb->TopicBlock.dwCount--;
|
|
}
|
|
|
|
pHeader->pTopic = pHeader->pLastTopic = pTopic;
|
|
|
|
// Update output offset
|
|
pEsi->ibBuf = (DWORD)(pCurPtr - pOutputBuffer);
|
|
|
|
return(S_OK);
|
|
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc INDEX
|
|
*
|
|
* @func HRESULT NEAR PASCAL | ProcessFiles |
|
|
* Sorts the file generated from the tree output into one
|
|
* list of sorted elements.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to index parameter block
|
|
*
|
|
* @rdesc S_OK, or errors if failed
|
|
*
|
|
* @notes
|
|
* This function processed the input buffers and uses dynamic
|
|
* memory allocation to sort each word as it come in. Once a
|
|
* word stops repeating, it is flush to disk and the memory is
|
|
* reset for the next word.
|
|
*************************************************************************/
|
|
|
|
HRESULT NEAR PASCAL ProcessFiles(_LPIPB lpipb, LPMERGEPARAMS lpmp)
|
|
{
|
|
// Local replacement variables
|
|
LPISI pIsi = &lpipb->isi;
|
|
LPESI pEsi = &lpipb->esi;
|
|
LPESB FAR * lrgPriorityQueue = pEsi->lrgPriorityQueue;
|
|
LONG uiQueueSize = pEsi->uiQueueSize;
|
|
LPB pQueuePtr;
|
|
WORD cNumOcc = lpipb->ucNumOccDataFields;
|
|
WORD OccSize = sizeof(OCCDATA) - sizeof(DWORD) + cNumOcc *
|
|
sizeof(DWORD);
|
|
int occf = lpipb->occf;
|
|
LPB pBufMax;
|
|
HANDLE hWord;
|
|
LPB lpbWord;
|
|
DWORD dwUniqueTerm = 0; // Used for calback function
|
|
#ifdef _DEBUG
|
|
BYTE astWord[300];
|
|
BYTE astLastWord[300];
|
|
#endif
|
|
|
|
// Working variables
|
|
PMERGEHEADER pHeader; // Pointer to merge header
|
|
LPESB pEsb; // Temp ESB pointer
|
|
PTOPICDATA pNewTopic; // Used to create new topic
|
|
DWORD loop; // Temp loop counter
|
|
HANDLE hHeader;
|
|
HFPB hOutputFile; // Handle to output file
|
|
int fRet; // Return value
|
|
USHORT uStringSize; // Size of Psacal String
|
|
ERRB errb;
|
|
PHRESULT phr = &errb;
|
|
|
|
static long Count = 0;
|
|
|
|
// Setup Block Manager
|
|
if ((fRet = ESBBlockAllocate (lpipb, lpipb->dwMemAllowed / 4)) != S_OK)
|
|
return(fRet);
|
|
|
|
// Allocate output buffer
|
|
if ((pEsi->hBuf = _GLOBALALLOC
|
|
(DLLGMEM_ZEROINIT, ESOUTPUT_BUFFER)) == NULL)
|
|
{
|
|
fRet = E_OUTOFMEMORY;
|
|
exit1:
|
|
return fRet;
|
|
}
|
|
|
|
pEsi->pOutputBuffer = (LPB)_GLOBALLOCK (pEsi->hBuf);
|
|
pEsi->ibBuf = 0;
|
|
|
|
// Create output file
|
|
GETTEMPFILENAME ((char)0, "eso", 0, pEsi->aszTempName);
|
|
|
|
if ((pEsi->hfpb = FileOpen(NULL, pEsi->aszTempName,
|
|
REGULAR_FILE, WRITE, &errb)) == NULL)
|
|
{
|
|
fRet = E_FILECREATE;
|
|
exit2:
|
|
FreeHandle (pEsi->hBuf);
|
|
pEsi->hBuf = NULL;
|
|
goto exit1;
|
|
}
|
|
hOutputFile = pEsi->hfpb;
|
|
|
|
// Setup new record in memory
|
|
if ((hHeader = _GLOBALALLOC
|
|
(DLLGMEM_ZEROINIT, sizeof (MERGEHEADER))) == NULL)
|
|
{
|
|
fRet = E_OUTOFMEMORY;
|
|
exit3:
|
|
FileClose (hOutputFile);
|
|
goto exit2;
|
|
}
|
|
pHeader = (PMERGEHEADER)_GLOBALLOCK (hHeader);
|
|
|
|
// Allocate buffer for a word, which include 64K + sizeof(WORD) + slack
|
|
if ((hWord = _GLOBALALLOC(DLLGMEM_ZEROINIT, 0x10004)) == NULL)
|
|
{
|
|
exit4:
|
|
_GLOBALUNLOCK(hHeader);
|
|
_GLOBALFREE (hHeader);
|
|
goto exit3;
|
|
}
|
|
|
|
pHeader->lpbWord = lpbWord = (LPB)_GLOBALLOCK(hWord);
|
|
|
|
#ifdef _DEBUG
|
|
astWord[0] = 0;
|
|
#endif
|
|
|
|
// Process all input buffers
|
|
do
|
|
{
|
|
DWORD dwWordLength;
|
|
DWORD dwFieldId;
|
|
LPB lpStart;
|
|
DWORD dwTopicCount;
|
|
|
|
#ifdef _DEBUG
|
|
Count++;
|
|
#endif
|
|
|
|
// Grab smallest record and send to buffer
|
|
pEsb = lrgPriorityQueue[1];
|
|
|
|
// Set the fill limit
|
|
pBufMax = pEsb->lrgbMem + pEsb->dwEsbSize - 256;
|
|
|
|
if ((pQueuePtr = pEsb->lrgbMem + pEsb->ibBuf) >= pBufMax)
|
|
{
|
|
if ((fRet = ESFillBuffer (lpipb, pEsb)) != S_OK)
|
|
goto exit4;
|
|
|
|
pQueuePtr = pEsb->lrgbMem;
|
|
}
|
|
|
|
// Save the record beginning
|
|
pQueuePtr += sizeof(DWORD);
|
|
lpStart = pQueuePtr;
|
|
|
|
// Get string
|
|
uStringSize = GETWORD ((LPUW)pQueuePtr) + sizeof (SHORT);
|
|
|
|
pQueuePtr += uStringSize;
|
|
#ifdef _DEBUG
|
|
if (pQueuePtr > pEsb->lrgbMem + pEsb->dwEsbSize)
|
|
SetErrCode (phr, E_ASSERT);
|
|
#endif
|
|
|
|
if (occf & OCCF_LENGTH)
|
|
pQueuePtr += CbByteUnpack (&dwWordLength, pQueuePtr);
|
|
else
|
|
dwWordLength = 0;
|
|
|
|
#ifdef _DEBUG
|
|
if (pQueuePtr >= pEsb->lrgbMem + pEsb->dwEsbSize)
|
|
SetErrCode (phr, E_ASSERT);
|
|
#endif
|
|
if (occf & OCCF_FIELDID)
|
|
pQueuePtr += CbByteUnpack (&dwFieldId, pQueuePtr);
|
|
else
|
|
dwFieldId = 0;
|
|
|
|
#ifdef _DEBUG
|
|
if (pQueuePtr > pEsb->lrgbMem + pEsb->dwEsbSize)
|
|
SetErrCode (phr, E_ASSERT);
|
|
#endif
|
|
// Is the word in the buffer equal to the new word?
|
|
// If it is not then flush the old word
|
|
if (*(LPUW)pHeader->lpbWord)
|
|
{
|
|
fRet = (StrCmp2BytePascal (pHeader->lpbWord, lpStart)
|
|
|| dwWordLength > pHeader->dwWordLength);
|
|
if (fRet == 0) // Same word, reduce the unique words count
|
|
lpipb->dwUniqueWord--;
|
|
if (fRet || dwFieldId > pHeader->dwFieldId)
|
|
{
|
|
#if defined(_DEBUG) && !defined(_MAC)
|
|
// Word out of order
|
|
if (StrCmp2BytePascal (pHeader->lpbWord, lpStart) > 0)
|
|
assert(FALSE);
|
|
#endif
|
|
if ((fRet = ESMemory2Disk (lpipb, pHeader, TRUE)) != S_OK)
|
|
return(fRet);
|
|
|
|
// Reset pHeader
|
|
MEMSET (pHeader, 0, sizeof (MERGEHEADER));
|
|
|
|
// Set the word buffer
|
|
pHeader->lpbWord = lpbWord;
|
|
#ifdef _DEBUG
|
|
STRCPY(astLastWord, astWord);
|
|
#endif
|
|
// Call the user callback every once in a while
|
|
if (!(++dwUniqueTerm % 8192L)
|
|
&& (lpipb->CallbackInfo.dwFlags & ERRFLAG_STATUS))
|
|
{
|
|
PFCALLBACK_MSG pCallbackInfo = &lpipb->CallbackInfo;
|
|
CALLBACKINFO Info;
|
|
|
|
Info.dwPhase = 2;
|
|
Info.dwIndex = (DWORD)((float)dwUniqueTerm / lpipb->dwUniqueWord * 100);
|
|
fRet = (*pCallbackInfo->MessageFunc)
|
|
(ERRFLAG_STATUS, pCallbackInfo->pUserData, &Info);
|
|
if (S_OK != fRet)
|
|
goto exit5;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Update the data
|
|
pHeader->dwFieldId = dwFieldId;
|
|
pHeader->dwWordLength = dwWordLength;
|
|
pHeader->dwStrLen = uStringSize;
|
|
|
|
// Copy word and header info
|
|
MEMCPY (pHeader->lpbWord, (LPB)lpStart, uStringSize);
|
|
#ifdef _DEBUG
|
|
if (uStringSize >= 300)
|
|
uStringSize = 300;
|
|
MEMCPY (astWord, lpStart + 2, uStringSize - 2);
|
|
astWord[uStringSize - 2] = 0;
|
|
//if (STRCMP(astWord, "87db") == 0)
|
|
// _asm int 3;
|
|
#endif
|
|
|
|
pQueuePtr += CbByteUnpack (&dwTopicCount, pQueuePtr);
|
|
pHeader->dwTopicCount += dwTopicCount;
|
|
|
|
#ifdef _DEBUG
|
|
if (pQueuePtr > pEsb->lrgbMem + pEsb->dwEsbSize)
|
|
SetErrCode (phr, E_ASSERT);
|
|
#endif
|
|
pNewTopic = NULL;
|
|
|
|
// Copy topic(s) to memory
|
|
for (loop = dwTopicCount; loop > 0; loop--)
|
|
{
|
|
DWORD dwTopicId;
|
|
|
|
// Get the topic id
|
|
pQueuePtr += CbByteUnpack (&dwTopicId, pQueuePtr);
|
|
|
|
// kevynct: if there is a to-delete list, and this topic is on it, skip it
|
|
if (lpmp && FindTopic(lpmp, dwTopicId))
|
|
{
|
|
// Get the occ count
|
|
if (cNumOcc)
|
|
{
|
|
DWORD dwOccCount;
|
|
DWORD dwT;
|
|
|
|
pQueuePtr += CbByteUnpack (&dwOccCount, pQueuePtr);
|
|
#ifdef _DEBUG
|
|
if (pQueuePtr > pEsb->lrgbMem + pEsb->dwEsbSize)
|
|
SetErrCode (phr, E_ASSERT);
|
|
#endif
|
|
for (; dwOccCount > 0; dwOccCount--)
|
|
{
|
|
// Fill up the buffer if run out of data
|
|
if (pQueuePtr >= pBufMax)
|
|
{
|
|
pEsb->ibBuf = (DWORD)(pQueuePtr - pEsb->lrgbMem);
|
|
if ((fRet = ESFillBuffer (lpipb, pEsb)) != S_OK)
|
|
goto exit5;
|
|
pQueuePtr = pEsb->lrgbMem;
|
|
}
|
|
|
|
switch (cNumOcc)
|
|
{
|
|
case 5:
|
|
pQueuePtr += CbByteUnpack (&dwT, pQueuePtr);
|
|
case 4:
|
|
pQueuePtr += CbByteUnpack (&dwT, pQueuePtr);
|
|
case 3:
|
|
pQueuePtr += CbByteUnpack (&dwT, pQueuePtr);
|
|
case 2:
|
|
pQueuePtr += CbByteUnpack (&dwT, pQueuePtr);
|
|
case 1:
|
|
pQueuePtr += CbByteUnpack (&dwT, pQueuePtr);
|
|
}
|
|
|
|
#ifdef _DEBUG
|
|
if (pQueuePtr > pEsb->lrgbMem + pEsb->dwEsbSize)
|
|
SetErrCode (phr, E_ASSERT);
|
|
#endif
|
|
} // end occ loop
|
|
} // end if occ non-zero
|
|
|
|
pHeader->dwTopicCount--;
|
|
continue;
|
|
} // end of to-delete condition
|
|
|
|
// Allocate a topicdata node
|
|
if ((pNewTopic == NULL) &&
|
|
(pNewTopic = GetBlockNode (&lpipb->TopicBlock)) == NULL)
|
|
{
|
|
if ((fRet = ESMemory2Disk(lpipb, pHeader, FLUSH_EXCEPT_LAST)) != S_OK)
|
|
{
|
|
exit5:
|
|
_GLOBALUNLOCK(hWord);
|
|
_GLOBALFREE(hWord);
|
|
goto exit4;
|
|
}
|
|
|
|
if ((pNewTopic = GetBlockNode (&lpipb->TopicBlock)) == NULL)
|
|
{
|
|
// Extremely weird, since we just release a bunch of
|
|
// memory
|
|
fRet = E_ASSERT;
|
|
goto exit5;
|
|
}
|
|
}
|
|
|
|
pNewTopic->dwTopicId = dwTopicId;
|
|
|
|
#ifdef _DEBUG
|
|
if (pQueuePtr > pEsb->lrgbMem + pEsb->dwEsbSize)
|
|
SetErrCode (phr, E_ASSERT);
|
|
#endif
|
|
// Set the other fields
|
|
pNewTopic->pOccData = pNewTopic->pLastOccData = NULL;
|
|
|
|
// Get the occ count
|
|
if (cNumOcc)
|
|
{
|
|
DWORD dwOccCount;
|
|
POCCDATA pOccData;
|
|
LPDW lpDw;
|
|
|
|
pQueuePtr += CbByteUnpack (&pNewTopic->dwOccCount,
|
|
pQueuePtr);
|
|
|
|
#ifdef _DEBUG
|
|
if (pQueuePtr > pEsb->lrgbMem + pEsb->dwEsbSize)
|
|
SetErrCode (phr, E_ASSERT);
|
|
#endif
|
|
for (dwOccCount = pNewTopic->dwOccCount; dwOccCount > 0;
|
|
dwOccCount--)
|
|
{
|
|
// Get all occ fields
|
|
if ((pOccData = (POCCDATA)GetBlockNode
|
|
(&lpipb->OccBlock)) == NULL )
|
|
{
|
|
if ((fRet = ESMemory2Disk(lpipb, pHeader,
|
|
FLUSH_EXCEPT_LAST)) != S_OK)
|
|
goto exit5;
|
|
|
|
if ((pOccData =
|
|
(POCCDATA)GetBlockNode(&lpipb->OccBlock)) == NULL)
|
|
{
|
|
// Extremely weird, since we just release a bunch of
|
|
// memory, unless there are so many duplicates of the same word
|
|
// in the topic
|
|
|
|
fRet = E_TOOMANYDUPS;
|
|
goto exit5;
|
|
}
|
|
}
|
|
|
|
// Fill up the buffer if run out of data
|
|
if (pQueuePtr >= pBufMax)
|
|
{
|
|
pEsb->ibBuf = (DWORD) (pQueuePtr - pEsb->lrgbMem);
|
|
if ((fRet = ESFillBuffer (lpipb, pEsb)) != S_OK)
|
|
goto exit5;
|
|
pQueuePtr = pEsb->lrgbMem;
|
|
}
|
|
|
|
lpDw = (LPDW)&pOccData->OccData;
|
|
switch (cNumOcc)
|
|
{
|
|
case 5:
|
|
pQueuePtr += CbByteUnpack (lpDw++, pQueuePtr);
|
|
case 4:
|
|
pQueuePtr += CbByteUnpack (lpDw++, pQueuePtr);
|
|
case 3:
|
|
pQueuePtr += CbByteUnpack (lpDw++, pQueuePtr);
|
|
case 2:
|
|
pQueuePtr += CbByteUnpack (lpDw++, pQueuePtr);
|
|
case 1:
|
|
pQueuePtr += CbByteUnpack (lpDw++, pQueuePtr);
|
|
}
|
|
|
|
#ifdef _DEBUG
|
|
if (pQueuePtr > pEsb->lrgbMem + pEsb->dwEsbSize)
|
|
SetErrCode (phr, E_ASSERT);
|
|
#endif
|
|
// Attach to the linked list
|
|
// Note that we are assumimg that the occurrences are
|
|
// already sorted, so no checking is done here
|
|
if (pNewTopic->pOccData == NULL)
|
|
{
|
|
pNewTopic->pLastOccData = pNewTopic->pOccData
|
|
= pOccData;
|
|
}
|
|
else
|
|
{
|
|
// Add to the end of the linked list
|
|
pNewTopic->pLastOccData->pNext = pOccData;
|
|
pNewTopic->pLastOccData = pOccData;
|
|
}
|
|
pOccData->pNext = NULL;
|
|
}
|
|
}
|
|
|
|
if (pNewTopic = MergeTopicNode (pHeader, pNewTopic, cNumOcc))
|
|
pHeader->dwTopicCount --;
|
|
}
|
|
|
|
// Update the offset
|
|
pEsb->ibBuf = (DWORD) (pQueuePtr - pEsb->lrgbMem);
|
|
|
|
// If next record doesn't fit in buffer
|
|
// Then reset to beginning and load data
|
|
if (pEsb->dwEsbSize - pEsb->ibBuf <= sizeof(DWORD) ||
|
|
pEsb->dwEsbSize - pEsb->ibBuf <= GETLONG((LPUL)pQueuePtr) +
|
|
2 * sizeof(DWORD))
|
|
{
|
|
if ((fRet = ESFillBuffer (lpipb, pEsb)) != S_OK)
|
|
goto exit4;
|
|
}
|
|
|
|
// Adjust priority queue
|
|
if (uiQueueSize > 1)
|
|
{
|
|
if (DwSubFo (pEsb->lfo, pEsb->lfoMax) != 0 &&
|
|
pEsb->ibBuf >= pEsb->dwEsbSize)
|
|
{
|
|
// Replace first record with last
|
|
lrgPriorityQueue[1] = lrgPriorityQueue[uiQueueSize];
|
|
lrgPriorityQueue[uiQueueSize] = NULL;
|
|
uiQueueSize--;
|
|
pEsi->uiQueueSize = uiQueueSize;
|
|
}
|
|
#if 0
|
|
else
|
|
{ // If the stream still has input add it back into the Queue
|
|
lrgPriorityQueue[uiQueueSize] = pEsb;
|
|
PQueueUp(lpipb, lrgPriorityQueue, uiQueueSize);
|
|
}
|
|
#endif
|
|
PQueueDown(lpipb); // Maintain sort order
|
|
}
|
|
else if (DwSubFo (pEsb->lfo, pEsb->lfoMax) != 0 &&
|
|
pEsb->ibBuf >= pEsb->dwEsbSize)
|
|
{
|
|
uiQueueSize--;
|
|
pEsi->uiQueueSize = uiQueueSize;
|
|
if ((fRet = ESMemory2Disk (lpipb, pHeader, FLUSH_NEW_RECORD)) != S_OK)
|
|
return(fRet);
|
|
}
|
|
} while (uiQueueSize);
|
|
|
|
fRet = ESFlushBuffer(pEsi);
|
|
goto exit5;
|
|
}
|
|
|
|
|
|
BOOL PASCAL NEAR FindTopic(LPMERGEPARAMS lpmp, DWORD dwTopicId)
|
|
{
|
|
register LPDW lpdw;
|
|
LPDW lpdwMac;
|
|
|
|
Assert(lpmp->dwCount > 0);
|
|
Assert(lpmp->lpTopicIdLast >= lpmp->rgTopicId);
|
|
Assert(lpmp->lpTopicIdLast < lpmp->rgTopicId + lpmp->dwCount);
|
|
|
|
if (lpmp->rgTopicId[0] > dwTopicId
|
|
||
|
|
*(lpdwMac = lpmp->rgTopicId + lpmp->dwCount - 1) < dwTopicId)
|
|
return FALSE;
|
|
|
|
if (*lpmp->lpTopicIdLast == dwTopicId)
|
|
return TRUE;
|
|
|
|
if (*lpmp->lpTopicIdLast > dwTopicId)
|
|
{
|
|
// re-start at the beginning
|
|
lpmp->lpTopicIdLast = lpmp->rgTopicId;
|
|
}
|
|
|
|
for (lpdw = lpmp->lpTopicIdLast; lpdw < lpdwMac + 1; lpdw++)
|
|
if (*lpdw == dwTopicId)
|
|
{
|
|
lpmp->lpTopicIdLast = lpdw;
|
|
return TRUE;
|
|
}
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc INTERNAL INDEXING
|
|
*
|
|
* @func int | CompareRecordBuffers |
|
|
* Called from PQueueUp/Down to sort the input buffers based first
|
|
* upon the string's, then TopicID's, then word length's, etc.
|
|
*
|
|
* @parm _LPIPB | lpipb |
|
|
* Pointer to the index parameter block
|
|
*
|
|
* @parm LPB | pBuffer A |
|
|
* Pointer to the first input buffer
|
|
*
|
|
* @parm LPB | pBuffer B |
|
|
* Pointer to the second input buffer
|
|
*
|
|
* @rdesc
|
|
* If pBufferA < pBufferB return < 0
|
|
* If pBufferA == pBufferB return = 0
|
|
* If pBufferA > pBufferB return > 0
|
|
*************************************************************************/
|
|
|
|
int PASCAL NEAR CompareRecordBuffers (_LPIPB lpipb, LPB pBufferA, LPB pBufferB)
|
|
{
|
|
// Local Replacement Variables
|
|
int occf = lpipb->occf;
|
|
int cNumOcc = lpipb->ucNumOccDataFields;
|
|
DWORD dwOccMin;
|
|
|
|
// Working Variables
|
|
int fRet;
|
|
int Len;
|
|
DWORD dwDataA;
|
|
DWORD dwDataB;
|
|
|
|
pBufferA += sizeof (DWORD); // Skip record length
|
|
pBufferB += sizeof (DWORD); // Skip record length
|
|
|
|
// Compare Pascal strings
|
|
if ((fRet = StrCmp2BytePascal(pBufferA, pBufferB)) != 0)
|
|
return fRet;
|
|
|
|
pBufferA += (Len = GETWORD ((LPUW)pBufferA) + sizeof (SHORT));
|
|
pBufferB += Len;
|
|
|
|
// Strings equal - compare FieldIds
|
|
// Compare Word Lengths
|
|
if (occf & OCCF_LENGTH)
|
|
{
|
|
pBufferA += CbByteUnpack (&dwDataA, pBufferA);
|
|
pBufferB += CbByteUnpack (&dwDataB, pBufferB);
|
|
if ((fRet = (int)(dwDataA - dwDataB)) != 0)
|
|
return fRet;
|
|
}
|
|
|
|
if (occf & OCCF_FIELDID)
|
|
{
|
|
pBufferA += CbByteUnpack (&dwDataA, pBufferA);
|
|
pBufferB += CbByteUnpack (&dwDataB, pBufferB);
|
|
if ((fRet = (int)(dwDataA - dwDataB)) != 0)
|
|
return fRet;
|
|
}
|
|
|
|
|
|
// Skip topic count
|
|
pBufferA += CbByteUnpack (&dwDataA, pBufferA);
|
|
pBufferB += CbByteUnpack (&dwDataB, pBufferB);
|
|
|
|
// Compare 1st topic Id
|
|
pBufferA += CbByteUnpack (&dwDataA, pBufferA);
|
|
pBufferB += CbByteUnpack (&dwDataB, pBufferB);
|
|
if ((fRet = (int)(dwDataA - dwDataB)) != 0)
|
|
return fRet;
|
|
|
|
// Get the occurrence count
|
|
pBufferA += CbByteUnpack (&dwDataA, pBufferA);
|
|
pBufferB += CbByteUnpack (&dwDataB, pBufferB);
|
|
|
|
if ((fRet = (int)(dwDataA - dwDataB)) < 0)
|
|
dwOccMin = dwDataA;
|
|
else
|
|
dwOccMin = dwDataB;
|
|
for (; dwOccMin; dwOccMin--)
|
|
{
|
|
switch (cNumOcc)
|
|
{
|
|
case 5:
|
|
pBufferA += CbByteUnpack (&dwDataA, pBufferA);
|
|
pBufferB += CbByteUnpack (&dwDataB, pBufferB);
|
|
if ((fRet = (int)(dwDataA - dwDataB)) != 0)
|
|
return fRet;
|
|
break;
|
|
case 4:
|
|
pBufferA += CbByteUnpack (&dwDataA, pBufferA);
|
|
pBufferB += CbByteUnpack (&dwDataB, pBufferB);
|
|
if ((fRet = (int)(dwDataA - dwDataB)) != 0)
|
|
return fRet;
|
|
break;
|
|
case 3:
|
|
pBufferA += CbByteUnpack (&dwDataA, pBufferA);
|
|
pBufferB += CbByteUnpack (&dwDataB, pBufferB);
|
|
if ((fRet = (int)(dwDataA - dwDataB)) != 0)
|
|
return fRet;
|
|
break;
|
|
case 2:
|
|
pBufferA += CbByteUnpack (&dwDataA, pBufferA);
|
|
pBufferB += CbByteUnpack (&dwDataB, pBufferB);
|
|
if ((fRet = (int)(dwDataA - dwDataB)) != 0)
|
|
return fRet;
|
|
break;
|
|
case 1:
|
|
pBufferA += CbByteUnpack (&dwDataA, pBufferA);
|
|
pBufferB += CbByteUnpack (&dwDataB, pBufferB);
|
|
if ((fRet = (int)(dwDataA - dwDataB)) != 0)
|
|
return fRet;
|
|
break;
|
|
}
|
|
}
|
|
return fRet;
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc INTERNAL INDEXING
|
|
*
|
|
* @func VOID | PQueueUp |
|
|
* The function restores the heap condition of a PQ, ie. the parent
|
|
* node must be less than the children. When the top node is inserted
|
|
* the heap condition may be violated if the resulting node
|
|
* is smaller than its parent. In this case the nodes have to
|
|
* be switched.
|
|
*
|
|
* @parm LPESI | lpesi |
|
|
* Pointer to external sort info, which contains all info
|
|
*
|
|
* @parm LONG | index |
|
|
* Index of the inserted node
|
|
*
|
|
*************************************************************************/
|
|
|
|
VOID PASCAL NEAR PQueueUp
|
|
(_LPIPB lpipb, LPESB FAR *lrgPriorityQueue, LONG index)
|
|
{
|
|
LPESB lpesbTemp; // Pointer to the inserted node
|
|
LPESB lpesbHalf; // Pointer to the parent node
|
|
WORD uiHalf; // Index of the parent's node
|
|
|
|
lpesbTemp = lrgPriorityQueue [index];
|
|
|
|
if ((uiHalf = (WORD) (index/2)) == 0)
|
|
return;
|
|
lpesbHalf = lrgPriorityQueue [uiHalf];
|
|
|
|
/* If the parent node is greated than the child, then exchange the
|
|
* nodes, The condition uiHalf != index makes sure that we stop
|
|
* at node 0 (top node)
|
|
*/
|
|
while (uiHalf && CompareRecordBuffers (lpipb, (LPB)lpesbHalf->lrgbMem +
|
|
lpesbHalf->ibBuf, (LPB)lpesbTemp->lrgbMem + lpesbTemp->ibBuf) > 0)
|
|
{
|
|
lrgPriorityQueue [index] = lpesbHalf;
|
|
index = uiHalf;
|
|
uiHalf = (WORD)(index/2);
|
|
lpesbHalf = lrgPriorityQueue [uiHalf];
|
|
}
|
|
lrgPriorityQueue[index] = lpesbTemp;
|
|
#if BINHN
|
|
SetQueue (&lpipb->esi);
|
|
#endif
|
|
}
|
|
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc INTERNAL INDEXING
|
|
*
|
|
* @func VOID | PQueueDown |
|
|
* The function restores the heap condition of a PQ, ie. the parent
|
|
* node must be less than the children. When the top node is removed
|
|
* the heap condition may be violated if the resulting node
|
|
* is greater than its children. In this case the nodes have to
|
|
* be switched.
|
|
*
|
|
* @parm LPESI | lpesi |
|
|
* Pointer to external sort info, which contains all info
|
|
*
|
|
*************************************************************************/
|
|
|
|
PRIVATE VOID PASCAL NEAR PQueueDown (_LPIPB lpipb)
|
|
{
|
|
LPESI lpesi = &lpipb->esi;
|
|
LPESB FAR *lrgPriorityQueue;
|
|
int CurIndex;
|
|
int ChildIndex;
|
|
int MaxCurIndex;
|
|
int MaxChildIndex;
|
|
LPESB lpesbSaved;
|
|
LPESB lpesbTemp;
|
|
LPESB lpesbChild;
|
|
|
|
lrgPriorityQueue = lpesi->lrgPriorityQueue;
|
|
lpesbSaved = lrgPriorityQueue[1];
|
|
MaxCurIndex = (MaxChildIndex = lpesi->uiQueueSize) / 2;
|
|
|
|
for (CurIndex = 1; CurIndex <= MaxCurIndex; CurIndex = ChildIndex)
|
|
{
|
|
// Get child index
|
|
ChildIndex = CurIndex * 2;
|
|
// Find the minimum of the two children
|
|
if (ChildIndex < MaxChildIndex)
|
|
{
|
|
if ((lpesbTemp = lrgPriorityQueue[ChildIndex + 1]) != NULL)
|
|
{
|
|
lpesbChild = lrgPriorityQueue[ChildIndex];
|
|
|
|
// The two children exist. Take the smallest
|
|
if (CompareRecordBuffers
|
|
(lpipb, (LPB)lpesbChild->lrgbMem + lpesbChild->ibBuf,
|
|
(LPB)lpesbTemp->lrgbMem + lpesbTemp->ibBuf) >= 0)
|
|
ChildIndex++;
|
|
}
|
|
}
|
|
|
|
// If the parent's node is less than the child, then break
|
|
// (heap condition met)
|
|
if (ChildIndex > MaxChildIndex)
|
|
break;
|
|
|
|
lpesbTemp = lrgPriorityQueue [ChildIndex];
|
|
|
|
if (CompareRecordBuffers (lpipb, (LPB)lpesbSaved->lrgbMem +
|
|
lpesbSaved->ibBuf, (LPB)lpesbTemp->lrgbMem+lpesbTemp->ibBuf) < 0)
|
|
break;
|
|
|
|
// Replace the node
|
|
lrgPriorityQueue [CurIndex] = lpesbTemp;
|
|
}
|
|
lrgPriorityQueue [CurIndex] = lpesbSaved;
|
|
#if _BINHN
|
|
SetQueue (lpesi);
|
|
#endif
|
|
}
|
|
|
|
|
|
PRIVATE PTOPICDATA PASCAL NEAR MergeTopicNode (PMERGEHEADER pHeader,
|
|
PTOPICDATA pNewTopic, int cNumOcc)
|
|
{
|
|
// PTOPICDATA pLastTopic;
|
|
PTOPICDATA pTopic, pPrevTopic;
|
|
int fResult;
|
|
|
|
if ((pTopic = pHeader->pLastTopic) == NULL)
|
|
{
|
|
// The list is empty
|
|
pHeader->pTopic = pHeader->pLastTopic = pNewTopic;
|
|
pNewTopic->pNext = NULL;
|
|
return(NULL);
|
|
|
|
}
|
|
|
|
fResult = pTopic->dwTopicId - pNewTopic->dwTopicId;
|
|
|
|
if (fResult < 0)
|
|
{
|
|
// New node. Add to the end
|
|
pNewTopic->pNext = NULL;
|
|
pHeader->pLastTopic->pNext = pNewTopic;
|
|
pHeader->pLastTopic = pNewTopic;
|
|
|
|
// Reset pNewTopic for next node allocation
|
|
return NULL;
|
|
}
|
|
|
|
if (fResult == 0)
|
|
{
|
|
// Same topic. Return pNewTopic for reuse
|
|
if (cNumOcc)
|
|
MergeOccurrence (pTopic, pNewTopic, cNumOcc);
|
|
return(pNewTopic);
|
|
}
|
|
|
|
// If we get to this point, the list is out of order
|
|
// Try to find the insertion point
|
|
pTopic = pHeader->pTopic;
|
|
pPrevTopic = NULL;
|
|
|
|
for (; pTopic->pNext; pTopic = pTopic->pNext)
|
|
{
|
|
if (pTopic->dwTopicId >= pNewTopic->dwTopicId)
|
|
{
|
|
/* We pass the inserted point */
|
|
break;
|
|
}
|
|
pPrevTopic = pTopic;
|
|
}
|
|
|
|
if (pTopic->dwTopicId == pNewTopic->dwTopicId)
|
|
{
|
|
// Same topic. Return pNewTopic for reuse
|
|
if (cNumOcc)
|
|
MergeOccurrence (pTopic, pNewTopic, cNumOcc);
|
|
return(pNewTopic);
|
|
}
|
|
|
|
// Handle empty case
|
|
if (pPrevTopic == NULL)
|
|
{
|
|
/* Insert at the beginning */
|
|
pNewTopic->pNext = pHeader->pTopic;
|
|
pHeader->pTopic = pNewTopic;
|
|
}
|
|
|
|
else
|
|
{
|
|
/* Inserted at the middle or the end */
|
|
pNewTopic->pNext = pPrevTopic->pNext;
|
|
pPrevTopic->pNext = pNewTopic;
|
|
}
|
|
|
|
// Update the last topic
|
|
while (pTopic->pNext)
|
|
{
|
|
pTopic = pTopic->pNext;
|
|
}
|
|
pHeader->pLastTopic = pTopic;
|
|
return(NULL);
|
|
}
|
|
|
|
/*************************************************************************
|
|
* @doc PRIVATE
|
|
* @func void | MergeOccurrence |
|
|
* Merge the occurrence by adding them in order
|
|
*************************************************************************/
|
|
PRIVATE VOID NEAR MergeOccurrence (PTOPICDATA pOldTopic,
|
|
PTOPICDATA pNewTopic, int cOccNum)
|
|
{
|
|
ERRB errb;
|
|
|
|
if (CompareOccurrence (&pOldTopic->pLastOccData->OccData[0],
|
|
&pNewTopic->pOccData->OccData[0], cOccNum) <= 0)
|
|
{
|
|
// The whole last list is less than the current list. This is
|
|
// what I expect
|
|
// We just linked the 2 lists together
|
|
pOldTopic->pLastOccData->pNext = pNewTopic->pOccData;
|
|
pOldTopic->pLastOccData = pNewTopic->pLastOccData;
|
|
pOldTopic->dwOccCount += pNewTopic->dwOccCount;
|
|
return;
|
|
}
|
|
|
|
// The current list is less than the old list.
|
|
// This is weird, but still we can handle it
|
|
if (CompareOccurrence (&pNewTopic->pOccData->OccData[0],
|
|
&pOldTopic->pOccData->OccData[0], cOccNum) <= 0)
|
|
{
|
|
pNewTopic->pLastOccData->pNext = pOldTopic->pOccData;
|
|
pOldTopic->pOccData = pNewTopic->pOccData;
|
|
pOldTopic->dwOccCount += pNewTopic->dwOccCount;
|
|
return;
|
|
}
|
|
|
|
SetErrCode (&errb, E_ASSERT);
|
|
}
|
|
|
|
/*====================================================================*/
|
|
#ifdef BINHN
|
|
PRIVATE VOID PASCAL NEAR SetQueue (LPESI pEsi)
|
|
{
|
|
unsigned int i = 0;
|
|
LPESB FAR *lrgPriorityQueue;
|
|
|
|
lrgPriorityQueue = pEsi->lrgPriorityQueue;
|
|
for (i = 0; i < 20 && i < pEsi->cesb ; i++)
|
|
{
|
|
if (lrgPriorityQueue[i])
|
|
pEsi->lpbQueueStr[i] = lrgPriorityQueue[i]->lrgbMem +
|
|
lrgPriorityQueue[i]->ibBuf + 6;
|
|
}
|
|
|
|
}
|
|
#endif
|
|
|
|
/************************************************************************
|
|
* @doc PRIVATE
|
|
* @func HRESULT PASCAL NEAR | ESBBlockAllocate |
|
|
* Set the memory allocation based on the memory of the machine
|
|
* @parm DWORD | lMemSize |
|
|
* Memory allocated for the indexer
|
|
* @rdesc S_OK, or E_OUTOFMEMORY
|
|
************************************************************************/
|
|
|
|
PRIVATE HRESULT PASCAL NEAR ESBBlockAllocate (_LPIPB lpipb, DWORD lMemSize)
|
|
{
|
|
DWORD dwTopicSize;
|
|
DWORD dwOccSize;
|
|
WORD OccNodeSize = sizeof (OCCDATA) - 1 + sizeof(DWORD) *
|
|
lpipb->ucNumOccDataFields; // About 24bytes
|
|
|
|
OccNodeSize = (OccNodeSize + 3) & ~3;
|
|
|
|
/* The memory is for topic block and occurrence blocks, which
|
|
* should be in the ratio 1:1.5
|
|
*/
|
|
dwTopicSize = (lMemSize * 2) / 5;
|
|
dwOccSize = lMemSize - dwTopicSize;
|
|
|
|
#if 0
|
|
/* Don't do anything if things are too small */
|
|
if (dwTopicSize < MAX_BLOCK_SIZE || dwOccSize < MAX_BLOCK_SIZE)
|
|
return(E_OUTOFMEMORY);
|
|
#endif
|
|
|
|
// Allocate a block manager for topic node
|
|
|
|
if ((lpipb->TopicBlock.pBlockMgr =
|
|
BlockInitiate ((MAX_BLOCK_SIZE * sizeof(TOPICDATA)/sizeof(TOPICDATA)),
|
|
sizeof(TOPICDATA),
|
|
(WORD)(dwTopicSize/MAX_BLOCK_SIZE),
|
|
USE_VIRTUAL_MEMORY | THREADED_ELEMENT)) == NULL)
|
|
{
|
|
exit2:
|
|
return SetErrCode (NULL, E_OUTOFMEMORY);
|
|
}
|
|
lpipb->TopicBlock.pFreeList =
|
|
(PLIST)BlockGetLinkedList(lpipb->TopicBlock.pBlockMgr);
|
|
|
|
// Allocate a block manager for occ node
|
|
if ((lpipb->OccBlock.pBlockMgr =
|
|
BlockInitiate((MAX_BLOCK_SIZE * OccNodeSize)/OccNodeSize,
|
|
OccNodeSize, (WORD)(lMemSize / MAX_BLOCK_SIZE),
|
|
USE_VIRTUAL_MEMORY | THREADED_ELEMENT)) == NULL)
|
|
{
|
|
BlockFree(lpipb->BTNodeBlock.pBlockMgr);
|
|
lpipb->BTNodeBlock.pBlockMgr = NULL;
|
|
goto exit2;
|
|
}
|
|
lpipb->OccBlock.pFreeList = (PLIST)BlockGetLinkedList(lpipb->OccBlock.pBlockMgr);
|
|
|
|
return (S_OK);
|
|
}
|
|
|
|
PRIVATE LPV PASCAL NEAR GetBlockNode (PBLKCOMBO pBlockCombo)
|
|
{
|
|
PLIST pList;
|
|
|
|
if (pBlockCombo->pFreeList == NULL)
|
|
{
|
|
if ((BlockGrowth (pBlockCombo->pBlockMgr) != S_OK))
|
|
return (NULL);
|
|
pBlockCombo->pFreeList =
|
|
(PLIST)BlockGetLinkedList(pBlockCombo->pBlockMgr);
|
|
}
|
|
pList = pBlockCombo->pFreeList;
|
|
pBlockCombo->pFreeList = pList->pNext;
|
|
pBlockCombo->dwCount ++;
|
|
// pList->pNext = NULL;
|
|
return (pList);
|
|
}
|
|
|
|
/*************************************************************************
|
|
*
|
|
* @doc INTERNAL
|
|
*
|
|
* @func BOOL FAR PASCAL | BuildIndexFile |
|
|
* This function is for debugging purpose only. In normal indexing,
|
|
* it will never be called. Since collecting words and indexing can
|
|
* take a long time, debugging the index phase can become a hassle that
|
|
* take several hours per shot. To minimize the index time for debugging,
|
|
* all the intermediate files are saved, which are:
|
|
* - the internal sorted result file, which contains all words and
|
|
* their occurrences sorted
|
|
* - the external sorted result file, which is a snap shot of the
|
|
* ESI structures and its ESB blocks
|
|
* The only steps left will be processing the occurrence list and doing
|
|
* permanent index
|
|
*
|
|
* To use the function, add the following lines in the app:
|
|
*
|
|
* extern HRESULT PASCAL FAR BuildIndexFile (LPSTR, LPSTR, LPSTR, WORD, WORD,
|
|
* WORD, INTERRUPT_FUNC, VOID FAR *, STATUS_FUNC, VOID FAR*, PHRESULT);
|
|
*
|
|
* int fDotest;
|
|
*
|
|
* if (fDotest) {
|
|
* return BuildIndexFile ((LPSTR)"c:/tmp/test.mvb!MVINDEX",
|
|
* (LPSTR)"c:/tmp/esi.tmp", (LPSTR)"c:/tmp/iso.tmp",
|
|
* OCCF_TOPICID, IDXF_NORMALIZE, 0, (INTERRUPT_FUNC)lpfnInterruptFunc,
|
|
* (LPV)NULL,
|
|
* (STATUS_FUNC)lpfnStatusFunc, (LPV)hwndGlobal,
|
|
* NULL);
|
|
* }
|
|
*
|
|
* @parm HFPB | hfpb |
|
|
* HFPB for index file if pstrIndexFile is NULL
|
|
*
|
|
* @parm LPB | pstrIndexFile |
|
|
* The .MVB + index file, usually with the format TEST.MVB!MVINDEX
|
|
*
|
|
* @parm LPB | lpbEsiFile |
|
|
* The external sort info file
|
|
*
|
|
* @parm LPB | lpbIsiFile |
|
|
* The internal sorted info filename
|
|
*
|
|
* @parm PINDEXINFO | pIndexInfo |
|
|
* IndexInfo
|
|
*
|
|
* @rdesc S_OK if succeeded, else other non-zero error codes
|
|
*************************************************************************/
|
|
|
|
PUBLIC HRESULT PASCAL EXPORT_API FAR BuildIndexFile
|
|
(HFPB hfpb, LPSTR pstrIndexFile,
|
|
LPB lpbEsiFile, LPB lpbIsiFile, PINDEXINFO pIndexInfo)
|
|
{
|
|
_LPIPB lpipb;
|
|
LPESI lpesi;
|
|
BOOL fRet;
|
|
ERRB errb;
|
|
DWORD loop;
|
|
FLOAT rLog;
|
|
BYTE bKeyIndex = 0;
|
|
|
|
if ((lpipb = MVIndexInitiate(pIndexInfo, NULL)) == NULL)
|
|
return E_FAIL;
|
|
|
|
lpesi = &lpipb->esi;
|
|
|
|
if (LoadEsiTemp (lpipb, lpesi, lpbEsiFile, lpbIsiFile, NULL) != S_OK)
|
|
{
|
|
fRet = E_FAIL;
|
|
exit0:
|
|
MVIndexDispose (lpipb);
|
|
return fRet;
|
|
}
|
|
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
// Allocate a huge buffer to contain all the sigma terms
|
|
if ((lpipb->wi.hSigma = _GLOBALALLOC (DLLGMEM_ZEROINIT,
|
|
(LCB)((lpipb->dwMaxTopicId + 1) * sizeof (SIGMA)))) == NULL)
|
|
return SetErrCode (&errb, E_OUTOFMEMORY);
|
|
lpipb->wi.hrgsigma = (HRGSIGMA)_GLOBALLOCK (lpipb->wi.hSigma);
|
|
|
|
// Small buffer containing pre-calculated values
|
|
if ((lpipb->wi.hLog = _GLOBALALLOC (DLLGMEM_ZEROINIT,
|
|
(CB)(cLOG_MAX * sizeof (FLOAT)))) == NULL)
|
|
{
|
|
SetErrCode (&errb, (HRESULT)(fRet = E_OUTOFMEMORY));
|
|
FreeHandle (lpipb->wi.hSigma);
|
|
goto exit0;
|
|
}
|
|
lpipb->wi.lrgrLog = (FLOAT FAR *)_GLOBALLOCK (lpipb->wi.hLog);
|
|
// Initialize the array
|
|
for (loop = cLOG_MAX - 1; loop > 0; --loop)
|
|
{
|
|
#ifndef ISBU_IR_CHANGE
|
|
rLog = (float) log10(cHundredMillion/(double)loop);
|
|
#else
|
|
rLog = (float)1.0 / (float)loop;
|
|
#endif // ISBU_IR_CHANGE
|
|
lpipb->wi.lrgrLog[loop] = rLog * rLog;
|
|
}
|
|
}
|
|
|
|
if ((fRet = MergeSortTreeFile (lpipb, NULL)) != S_OK)
|
|
return SetErrCode (&errb, (HRESULT)fRet);
|
|
if ((lpipb->idxf & KEEP_TEMP_FILE) == 0)
|
|
FileUnlink (NULL, lpipb->isi.aszTempName, REGULAR_FILE);
|
|
|
|
// If we are doing term-weighting we have to square root all sigma values
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
// ISBU_IR_CHANGE not necessary 'cos sqrt computation is necessary in both cases
|
|
for (loop = 0; loop < lpipb->dwMaxTopicId + 1; ++loop)
|
|
lpipb->wi.hrgsigma[loop] =
|
|
(float)sqrt ((double)lpipb->wi.hrgsigma[loop]);
|
|
}
|
|
|
|
// Analyze data to get the best compression scheme
|
|
// TopicId
|
|
VGetBestScheme(&lpipb->cKey[CKEY_TOPIC_ID],
|
|
&lpipb->BitCount[CKEY_TOPIC_ID][0], lcbitBITSTREAM_ILLEGAL, TRUE);
|
|
// Occurrence Count
|
|
VGetBestScheme(&lpipb->cKey[CKEY_OCC_COUNT],
|
|
&lpipb->BitCount[CKEY_OCC_COUNT][0], lcbitBITSTREAM_ILLEGAL, TRUE);
|
|
|
|
if (lpipb->occf & OCCF_COUNT)
|
|
{
|
|
VGetBestScheme(&lpipb->cKey[bKeyIndex],
|
|
&lpipb->BitCount[bKeyIndex][0], lcbitBITSTREAM_ILLEGAL, TRUE);
|
|
bKeyIndex++;
|
|
}
|
|
|
|
if (lpipb->occf & OCCF_OFFSET)
|
|
{
|
|
VGetBestScheme(&lpipb->cKey[bKeyIndex],
|
|
&lpipb->BitCount[bKeyIndex][0], lcbitBITSTREAM_ILLEGAL, TRUE);
|
|
bKeyIndex++;
|
|
}
|
|
|
|
// Call the user callback every once in a while
|
|
if (lpipb->CallbackInfo.dwFlags & ERRFLAG_STATUS)
|
|
{
|
|
PFCALLBACK_MSG pCallbackInfo = &lpipb->CallbackInfo;
|
|
CALLBACKINFO Info;
|
|
|
|
Info.dwPhase = 2;
|
|
Info.dwIndex = 100;
|
|
fRet = (*pCallbackInfo->MessageFunc)
|
|
(ERRFLAG_STATUS, pCallbackInfo->pUserData, &Info);
|
|
if (S_OK != fRet)
|
|
goto exit0;
|
|
}
|
|
|
|
// Build the permanent index
|
|
fRet = BuildBTree(NULL, lpipb, lpipb->esi.aszTempName, hfpb, pstrIndexFile);
|
|
if (lpipb->idxf & IDXF_NORMALIZE)
|
|
{
|
|
FreeHandle (lpipb->wi.hLog);
|
|
FreeHandle (lpipb->wi.hSigma);
|
|
}
|
|
goto exit0;
|
|
}
|
|
|
|
PRIVATE VOID PASCAL NEAR SaveEsiTemp (_LPIPB lpipb, LPESI lpesi)
|
|
{
|
|
GHANDLE hfpb;
|
|
LPESB lpesb;
|
|
char szEsi[100];
|
|
|
|
GETTEMPFILENAME ((char)0, "foo", 0, szEsi);
|
|
if ((hfpb = FileOpen(NULL, szEsi, REGULAR_FILE, READ_WRITE, NULL)) == NULL)
|
|
return;
|
|
|
|
FileWrite(hfpb, lpipb, sizeof(IPB), NULL);
|
|
|
|
for (lpesb = lpesi->lpesbRoot; lpesb; lpesb = lpesb->lpesbNext)
|
|
{
|
|
if (FileWrite(hfpb, lpesb, sizeof(ESB), NULL) != sizeof(ESB))
|
|
{
|
|
FileClose (hfpb);
|
|
FileUnlink (NULL, szEsi, REGULAR_FILE);
|
|
return;
|
|
}
|
|
}
|
|
FileClose (hfpb);
|
|
MEMCPY (lpipb->szEsiTemp, szEsi, 100);
|
|
}
|
|
|
|
PRIVATE VOID PASCAL NEAR UpdateEsiTemp (_LPIPB lpipb)
|
|
{
|
|
GHANDLE hfpb;
|
|
|
|
if ((hfpb = FileOpen(NULL, lpipb->szEsiTemp, REGULAR_FILE,
|
|
READ_WRITE, NULL)) == NULL)
|
|
return;
|
|
|
|
FileWrite(hfpb, lpipb, sizeof(IPB), NULL);
|
|
FileClose (hfpb);
|
|
}
|
|
|
|
PRIVATE BOOL PASCAL LoadEsiTemp (_LPIPB lpipb, LPESI lpesi, LPB lpbEsiFile,
|
|
LPB lpbIsiFile, PHRESULT phr)
|
|
{
|
|
LPESB lpesb;
|
|
HFILE hFile;
|
|
ESB esb;
|
|
HANDLE hesb;
|
|
HRESULT fRet;
|
|
IPB ipb;
|
|
LPISI pIsi = &lpipb->isi; // Pointer to internal sort info
|
|
|
|
/* Copy the internal sort info filename */
|
|
MEMCPY (pIsi->aszTempName, lpbIsiFile, lstrlen(lpbIsiFile));
|
|
|
|
/* Read in the external sort buffer info */
|
|
|
|
if ((hFile = _lopen (lpbEsiFile, READ)) == HFILE_ERROR)
|
|
return E_NOTEXIST;
|
|
|
|
/* Read old IPB info */
|
|
_lread (hFile, &ipb, sizeof(IPB));
|
|
|
|
/* Transfer meaningful data */
|
|
|
|
lpipb->dwIndexedWord = ipb.dwIndexedWord;
|
|
lpipb->dwUniqueWord = ipb.dwUniqueWord;
|
|
lpipb->dwByteCount = ipb.dwByteCount;
|
|
lpipb->dwOccOffbits = ipb.dwOccOffbits;
|
|
lpipb->dwOccExtbits = ipb.dwOccExtbits;
|
|
lpipb->dwMaxFieldId = ipb.dwMaxFieldId;
|
|
lpipb->dwMaxWCount = ipb.dwMaxWCount;
|
|
lpipb->dwMaxOffset = ipb.dwMaxOffset;
|
|
lpipb->dwTotal3bWordLen = ipb.dwTotal3bWordLen;
|
|
lpipb->dwTotal2bWordLen = ipb.dwTotal2bWordLen;
|
|
lpipb->dwTotalUniqueWordLen = ipb.dwTotalUniqueWordLen;
|
|
lpipb->lcTopics = ipb.lcTopics;
|
|
lpipb->dwMaxTopicId = ipb.dwMaxTopicId;
|
|
// lpipb->dwMemAllowed = ipb.dwMemAllowed;
|
|
lpipb->dwMaxRecordSize = ipb.dwMaxRecordSize;
|
|
lpipb->dwMaxEsbRecSize = ipb.dwMaxEsbRecSize;
|
|
lpipb->dwMaxWLen = ipb.dwMaxWLen;
|
|
lpipb->idxf = ipb.idxf;
|
|
|
|
while ((_lread (hFile, &esb, sizeof(ESB))) == sizeof(ESB))
|
|
{
|
|
if ((hesb = _GLOBALALLOC(GMEM_MOVEABLE | GMEM_ZEROINIT,
|
|
sizeof(ESB))) == NULL) {
|
|
|
|
fRet = SetErrCode (phr,E_OUTOFMEMORY);
|
|
exit0:
|
|
_lclose (hFile);
|
|
return fRet;
|
|
}
|
|
|
|
lpesb = (LPESB)_GLOBALLOCK (hesb);
|
|
|
|
/* Copy the ESB information */
|
|
*lpesb = esb;
|
|
|
|
/* Update the structure */
|
|
lpesb->hStruct = hesb;
|
|
|
|
lpesb->lpesbNext = lpesi->lpesbRoot;
|
|
lpesi->lpesbRoot= lpesb;
|
|
lpesi->cesb ++;
|
|
}
|
|
_lclose (hFile);
|
|
|
|
fRet = S_OK;
|
|
goto exit0;
|
|
|
|
}
|
|
|
|
HRESULT FAR PASCAL AllocSigmaTable (_LPIPB lpipb)
|
|
{
|
|
ERRB errb;
|
|
DWORD loop;
|
|
float rLog;
|
|
|
|
|
|
if ((lpipb->wi.hSigma = _GLOBALALLOC (DLLGMEM_ZEROINIT,
|
|
(LCB)((lpipb->dwMaxTopicId + 1) * sizeof (SIGMA)))) == NULL)
|
|
return SetErrCode (&errb, E_OUTOFMEMORY);
|
|
lpipb->wi.hrgsigma = (HRGSIGMA)_GLOBALLOCK (lpipb->wi.hSigma);
|
|
|
|
if ((lpipb->wi.hLog = _GLOBALALLOC (DLLGMEM_ZEROINIT,
|
|
(CB)(cLOG_MAX * sizeof (FLOAT)))) == NULL)
|
|
{
|
|
FreeHandle (lpipb->wi.hSigma);
|
|
return SetErrCode (&errb, E_OUTOFMEMORY);
|
|
}
|
|
lpipb->wi.lrgrLog = (FLOAT FAR *)_GLOBALLOCK (lpipb->wi.hLog);
|
|
// Initialize the array
|
|
for (loop = cLOG_MAX - 1; loop > 0; --loop)
|
|
{
|
|
#ifndef ISBU_IR_CHANGE
|
|
rLog = (float) log10(cHundredMillion/(double)loop);
|
|
#else
|
|
rLog = (float)1.0 / (float)loop;
|
|
#endif // ISBU_IR_CHANGE
|
|
lpipb->wi.lrgrLog[loop] = rLog * rLog;
|
|
}
|
|
return(S_OK);
|
|
}
|