//+--------------------------------------------------------------------------- // // Microsoft Windows // Copyright (C) Microsoft Corporation, 1991 - 2000. // // File: RESMAN.CXX // // Contents: Resource Manager // // Classes: CResManager // // History: 08-Apr-91 BartoszM Created // 4-Jan-95 BartoszM Separated Filter Manager // Jan-08-97 mohamedn CFwEventItem and CDmFwEventItem // 24-Feb-97 SitaramR Push filtering // //---------------------------------------------------------------------------- #include #pragma hdrstop #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "resman.hxx" #include "ci.hxx" #include "partn.hxx" #include "pindex.hxx" #include "mindex.hxx" #include "idxids.hxx" #include "indxact.hxx" #include "merge.hxx" #include "mmerglog.hxx" #include "pendcur.hxx" #include "fltstate.hxx" #include "idle.hxx" #include "notxact.hxx" #include "lowres.hxx" const ULONG lowDiskWaterMark = 3 * 512 * 1024; // 1.5 MB const ULONG highDiskWaterMark = lowDiskWaterMark + 512 * 1024; // 2.0 MB const ULONG minDiskFreeForMerge = lowDiskWaterMark; class CRevertBoolValue { public: CRevertBoolValue( BOOL & rfValue, BOOL fValue ) : _rfVal( rfValue ), _fRevert( TRUE ) { Win4Assert( rfValue != fValue ); _fPrevVal = _rfVal; _rfVal = fValue; } ~CRevertBoolValue() { if (_fRevert) _rfVal = _fPrevVal; } void Revert() { _rfVal = _fPrevVal; _fRevert = FALSE; } private: BOOL & _rfVal; BOOL _fPrevVal; BOOL _fRevert; }; CMergeThreadKiller::CMergeThreadKiller( CResManager & resman ) : _resman(resman), _fKill(TRUE) { } CMergeThreadKiller::~CMergeThreadKiller() { if ( _fKill ) { _resman._fStopMerge = TRUE; _resman._thrMerge.Resume(); _resman.StopMerges(); } } //+--------------------------------------------------------------------------- // // Class: CPendQueueTrans // // Purpose: Transaction for flushing the pending queue if there is a // failure while extracting entries out of the pending queue // and adding them to the changelog. // // History: 9-11-95 srikants Created // //---------------------------------------------------------------------------- class CPendQueueTrans : public CTransaction { public: CPendQueueTrans( CPendingQueue & pendQueue ) : _pendQueue( pendQueue ) { } ~CPendQueueTrans() { if ( CTransaction::XActAbort == _status ) { CLock lock( _pendQueue.GetMutex() ); _pendQueue.LokFlushCompletedEntries(); } } private: CPendingQueue & _pendQueue; }; //+--------------------------------------------------------------------------- // // Member: CResManager::CResManager, public // // Arguments: [cat] -- catalog // [xact] -- transaction // // History: 08-Apr-91 BartoszM Created // Jan-07-96 mohamedn CFwEventItem // //---------------------------------------------------------------------------- CResManager::CResManager( PStorage &storage, CCiFrameworkParams & params, ICiCDocStore * pDocStore, CI_STARTUP_INFO const & startupInfo, IPropertyMapper * pPropMapper, CTransaction& xact, XInterface & xIndexNotifTable ) : _sigResman(eSigResman), _frmwrkParams( params ), _mergeTime(0), _storage ( storage ), _sKeyList(0), _idxTab( _storage.QueryIndexTable( xact ) ), _partList ( storage, *_idxTab, _sKeyList, xact, params ), _fresh ( storage, xact, _partList ), _partidToMerge ( partidInvalid ), _fStopMerge(FALSE), _pMerge(0), #pragma warning( disable : 4355 ) // this used in base initialization _thrMerge ( MergeThread, this, TRUE ), // create suspended _mergeKiller( *this ), #pragma warning( default : 4355 ) _activeDeletedIndex( _idxTab->GetDeletedIndex() ), _cQuery(0), _cFilteredDocuments(0), _pBackupWorkItem(0), _isBeingEmptied(FALSE), _isLowOnDiskSpace(FALSE), _isDismounted(FALSE), _isOutOfDate(FALSE), _isCorrupt(FALSE), _fFirstTimeUpdatesAreEnabled( TRUE ), _fPushFiltering( FALSE ), _fFlushWorkerActive( FALSE ), _configFlags( startupInfo.startupFlags ), _xIndexNotifTable( xIndexNotifTable.Acquire() ), _dwFilteringState( 0 ), _pFilterAgent( 0 ) { Win4Assert( 0 != pDocStore ); // // Look for a resource monitor. If none exists, use the default. // _xLowResDefault.Set( new CLowRes(_frmwrkParams) ); SCODE sc = pDocStore->QueryInterface( IID_ICiCResourceMonitor, _xLowRes.GetQIPointer() ); if ( FAILED(sc) ) { _xLowResDefault->AddRef(); _xLowRes.Set( _xLowResDefault.GetPointer() ); } // // // The presence or absence of an actual pointer in _xIndexNotifTable, // indicates whether the client has specified pull filtering or push // filtering. // _fPushFiltering = !_xIndexNotifTable.IsNull(); // // Initialize the resman in all the partitions. // CPartIter iter; for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList)) { CPartition * pPart = iter.LokGet(); pPart->SetResMan( this, FPushFiltering() ); } if ( _fPushFiltering ) { // // In push filtering, an identity translator is used // CIdentityNameTranslator *pIdentityTrans = new CIdentityNameTranslator(); sc = pIdentityTrans->QueryInterface( IID_ICiCDocNameToWorkidTranslator, _translator.GetQIPointer() ); Win4Assert( SUCCEEDED( sc ) ); _translator->Release(); // QI does an AddRef } else { // // In pull filtering, the client provides the translator, which may // be the Ex version. // sc = pDocStore->QueryInterface( IID_ICiCDocNameToWorkidTranslatorEx, _translator2.GetQIPointer() ); if ( SUCCEEDED(sc) ) sc = _translator2->QueryInterface( IID_ICiCDocNameToWorkidTranslator, _translator.GetQIPointer() ); else sc = pDocStore->QueryInterface( IID_ICiCDocNameToWorkidTranslator, _translator.GetQIPointer() ); } if ( S_OK != sc ) THROW( CException(sc) ); // // Create a workIdToDocName converter object. // _xWorkidToDocName.Set( new CWorkIdToDocName( _translator.GetPointer(), _translator2.GetPointer() ) ); ICiCAdviseStatus * pAdviseStatus; sc = pDocStore->QueryInterface( IID_ICiCAdviseStatus, (void **) & pAdviseStatus ); if ( S_OK != sc ) THROW( CException(sc) ); _adviseStatus.Set( pAdviseStatus ); _prfCounter.SetAdviseStatus( pAdviseStatus ); ICiCPropertyStorage * pPropStore; sc = pDocStore->QueryInterface( IID_ICiCPropertyStorage, (void **) & pPropStore ); if ( S_OK != sc ) THROW( CException(sc) ); _propStore.Set( pPropStore ); pDocStore->AddRef(); _docStore.Set( pDocStore ); pPropMapper->AddRef(); _mapper.Set( pPropMapper ); ULONG mergeTime = _frmwrkParams.GetMasterMergeTime(); _mergeTime = CMasterMergePolicy::ComputeMidNightMergeTime(mergeTime); if ( -1 == _mergeTime ) { CFwEventItem item(EVENTLOG_ERROR_TYPE, MSG_CI_BAD_SYSTEM_TIME, 0); item.ReportEvent( _adviseStatus.GetReference() ); THROW( CException( STATUS_INVALID_PARAMETER ) ); } // // Restore the information associated with any master merge that // was stopped when the drive was dismounted. // _partList.RestoreMMergeState(*this, _storage); _partIter.LokInit( _partList ); _isLowOnDiskSpace = LokCheckIfDiskLow( *this, _storage, _isLowOnDiskSpace, _adviseStatus.GetReference() ); _thrMerge.SetPriority ( _frmwrkParams.GetThreadPriorityMerge() ); _mergeKiller.Defuse(); _thrMerge.Resume(); // // Set the merge progress indicator to 0, and refresh other counters // _prfCounter.Update( CI_PERF_MERGE_PROGRESS, 0 ); LokUpdateCounters(); // // Enable or disables updates/notifications based on disk space available // and whether the catalog is readonly // if ( _isLowOnDiskSpace || _storage.IsReadOnly() ) { DisableUpdates( partidDefault ); _state.LokSetState( eUpdatesDisabled ); } else { EnableUpdates( partidDefault ); SCODE sc = _docStore->EnableUpdates(); if ( SUCCEEDED( sc ) ) _state.LokSetState( eSteady ); else _state.LokSetState( eUpdatesToBeEnabled ); } } //+------------------------------------------------------------------------- // // Member: CResManager::~CResManager, public // // Synopsis: Shuts down merge(s) // // History: 13-Aug-93 KyleP Created // //-------------------------------------------------------------------------- CResManager::~CResManager() { _workMan.AbortWorkItems(); LokDeleteFlushWorkItems(); _workMan.WaitForDeath(); StopMerges(); #if CIDBG==1 // ====================================== { CPriLock lock(_mutex); Win4Assert( 0 == _pBackupWorkItem ); } // ====================================== #endif // CIDBG==1 } //+--------------------------------------------------------------------------- // // Member: CResManager::MergeThread, public // // Synopsis: Entry point for the thread responsible // for asynchronous merges // // History: 05-Mar-92 BartoszM Created // //---------------------------------------------------------------------------- DWORD WINAPI CResManager::MergeThread( void* self ) { if ( !((CResManager*)self)->_fStopMerge ) ((CResManager*)self)->DoMerges(); return 0; } //============== // STATE CHANGES //============== //+--------------------------------------------------------------------------- // // Function: Dismount // // Synopsis: Dismounts the catalog by stopping any in-progress merge // and also finishing off any pending writes ( eg. ChangeLog). // // History: 6-20-94 srikants Created // //---------------------------------------------------------------------------- NTSTATUS CResManager::Dismount() { ciDebugOut(( DEB_ITRACE, "*** CI: Initiating DisMount ***\n" )); NTSTATUS status = STATUS_SUCCESS; TRY { // // Inform all the partitions to cleanup. We have to abort any // in-progress merges without having to take a lock. There could // be another thread (like the filter thread) with the lock doing // a long operation. In one case we had it trying to create a new // fresh test and the memory writer could not make any progress // because the merge was continuing to dirty the pages. // // There is no need to take a lock because the partitions are not // going away and the operation we are about to do is just turning // on a one-way flag. // { PARTITIONID partid = 1; CPartition* pPart = _partList.LokGetPartition ( partid ); pPart->PrepareForCleanup(); } // // Stop in progress merges. // StopMerges(); // // Dismount each of the partitions. // // ================================================ { CPriLock lock( _mutex ); Win4Assert( !_isDismounted ); _isDismounted = TRUE; CKeyList * pKeyList = _sKeyList.Acquire(); delete pKeyList; CPartIter iter; for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList)) { CPartition * pPart = iter.LokGet(); // ======================================================= CChangeTrans xact( *this, pPart ); if ( STATUS_SUCCESS == pPart->LokDismount( xact ) ) { xact.LokCommit(); } //========================================================= } if ( _pBackupWorkItem ) _pBackupWorkItem->GetSaveProgressTracker().SetAbort(); } // ================================================ _workMan.WaitForDeath(); } CATCH( CException, e ) { status = e.GetErrorCode(); ciDebugOut(( DEB_ERROR, "ContentIndex Dismount Failed. Error Code 0x%X\n", status )); } END_CATCH ciDebugOut(( DEB_ITRACE, "*** CI: Completed DisMount ***\n" )); return status; } //+--------------------------------------------------------------------------- // // Function: CResManager::Empty, public // // Synopsis: Releases resources associated with the resman object. This // includes all indexes, the persistent freshlog, the persistant // changelog, the freshtest, and the master merge log. // // History: 15-Aug-94 DwightKr Created // //---------------------------------------------------------------------------- void CResManager::Empty() { CPriLock lock( _mutex ); _isBeingEmptied = TRUE; // // Cancel any connections to the CI Filter service. // _pFilterAgent->LokCancel(); // // Delete everything from the index table here first. // _idxTab->LokEmpty(); // // If anything fails after this point, chkdsk /f or autochk will // release the disk storage associated with the leaked objects // since they are no longer part of the persistent index list. // // // For each partition, zombify all indexes. // CPartIter iter; for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList)) { CPartition* pPart = iter.LokGet(); // // Zombify all indexes in this partition, and delete them if their // ref-count is 0. // CIndex ** aIndex = 0; TRY { unsigned cIndex; aIndex = pPart->LokZombify( cIndex ); ReleaseIndexes( cIndex, aIndex, NULL ); } CATCH ( CException, e ) { } END_CATCH delete [] aIndex; // // Delete the change log associated with this partition. These // routines do not throw exceptions. // WORKID widChangeLog = _partList.GetChangeLogObjectId( pPart->GetId() ); if ( widChangeLog != widInvalid) _storage.RemoveObject( widChangeLog ); WORKID widMMergeLog; WORKID widDummy; pPart->GetMMergeObjectIds( widMMergeLog, widDummy, widDummy ); if ( widMMergeLog != widInvalid) _storage.RemoveObject( widMMergeLog ); } _fresh.LokEmpty(); // Release storage associated with fresh test WORKID widFreshLog = _storage.GetSpecialItObjectId( itFreshLog ); #ifdef KEYLIST_ENABLED WORKID widKeyList = _storage.GetSpecialItObjectId( itMMKeyList ); #else WORKID widKeyList = widInvalid; #endif // WORKID widPhrLat = _storage.GetSpecialItObjectId( itPhraseLat ); if ( widFreshLog != widInvalid) _storage.RemoveObject( widFreshLog ); if ( widKeyList != widInvalid) _storage.RemoveObject( widKeyList ); if ( widPhrLat != widInvalid) _storage.RemoveObject( widPhrLat ); } //+--------------------------------------------------------------------------- // // Function: StopCurrentMerge // // Synopsis: Aborts any in-progress merge. // // History: 5-04-94 srikants Created // //---------------------------------------------------------------------------- NTSTATUS CResManager::StopCurrentMerge() { { CPriLock lock( _mutex ); // // If we're doing a merge right now then kill it off. // if ( 0 != _pMerge ) _pMerge->LokAbort(); } return STATUS_SUCCESS; } //+--------------------------------------------------------------------------- // // Member: CResManager::DisableUpdates, public // // Arguments: [partid] -- partition id // // History: 24-Dec-94 Anonymous Created // //---------------------------------------------------------------------------- void CResManager::DisableUpdates( PARTITIONID partid ) { CPriLock lock( _mutex ); CPartition * pPart = _partList.LokGetPartition( partid ); Win4Assert( 0 != pPart ); pPart->LokDisableUpdates(); } //+--------------------------------------------------------------------------- // // Member: CResManager::EnableUpdates, public // // Arguments: [partid] -- partition id // // History: 24-Dec-94 Anonymous Created // //---------------------------------------------------------------------------- void CResManager::EnableUpdates( PARTITIONID partid ) { CPriLock lock( _mutex ); CPartition * pPart = _partList.LokGetPartition( partid ); Win4Assert( 0 != pPart ); pPart->LokEnableUpdates( _fFirstTimeUpdatesAreEnabled ); _fFirstTimeUpdatesAreEnabled = FALSE; } //+--------------------------------------------------------------------------- // // Function: LokCheckIfDiskLow // // Synopsis: Checks if we are running low on free disk space. It has // a "LowWaterMark" and a "HighWaterMark". // // If current state is "not full", then it will check to // see if the free disk space is < the LowWaterMark. // // if current state is "full", then it will check to see if // the free disk space is < the HighWaterMark. // // The HighWaterMark is > the LowWaterMark to prevent hysterisis. // // Arguments: [resman] -- Resource manager // [storage] -- Storage object. // [fCurrent] -- Current state. // [adviseStatus] -- reference to ICiCAdviseStatus // // Returns: TRUE if we are running low on disk space. // FALSE otherwise. // // History: 10-11-94 srikants Created // Jan-07-96 mohamedn CFwEventItem // //---------------------------------------------------------------------------- BOOL LokCheckIfDiskLow( CResManager & resman, PStorage & storage, BOOL fIsCurrentFull, ICiCAdviseStatus & adviseStatus ) { BOOL fLowOnDisk = fIsCurrentFull; TRY { __int64 diskTotal, diskRemaining; storage.GetDiskSpace( diskTotal, diskRemaining ); if ( !fIsCurrentFull ) fLowOnDisk = diskRemaining < lowDiskWaterMark; else fLowOnDisk = diskRemaining < highDiskWaterMark; // // It is okay to read it without mutex as it is only a heuristic. // if ( fLowOnDisk && !fIsCurrentFull && !storage.IsReadOnly() ) { ciDebugOut(( DEB_WARN, "****YOU ARE RUNNING LOW ON DISK SPACE****\n")); ciDebugOut(( DEB_WARN, "****PLEASE FREE UP SOME SPACE ****\n")); ULONG mbToFree = highDiskWaterMark/(1024*1024); ULONG mbIndex; resman.IndexSize( mbIndex ); mbToFree = max( mbToFree, mbIndex ); mbToFree = min( mbToFree, 50 ); // don't ask for more than 50 MB CFwEventItem item( EVENTLOG_AUDIT_FAILURE, MSG_CI_LOW_DISK_SPACE, 2); item.AddArg(storage.GetVolumeName() ); item.AddArg(mbToFree); item.ReportEvent(adviseStatus); } else if ( fIsCurrentFull && !fLowOnDisk ) { ciDebugOut(( DEB_WARN, "****DISK SPACE FREED UP ****\n")); } } CATCH( CException,e ) { ciDebugOut(( DEB_ERROR, "Error 0x%X while getting disk space info\n", e.GetErrorCode() )); } END_CATCH return ( fLowOnDisk ); } //+--------------------------------------------------------------------------- // // Function: NoFailFreeResources // // Synopsis: deletes a big wordlist and schedules refiltering // // History: 6-Jan-95 BartoszM Created // //---------------------------------------------------------------------------- void CResManager::NoFailFreeResources() { ciDebugOut(( DEB_WARN, "Running out of resources. " )); TRY { PARTITIONID partId = 1; CPriLock lock(_mutex); CPartition * pPart = LokGetPartition( partId ); // // Before calling remove word list, we must append any refiled // documents to the doc queue. This is because, the docqueue // can handle only one set of refiled documents. // { // ========================================= CChangeTrans xact( *this, pPart ); pPart->LokAppendRefiledDocs( xact ); xact.LokCommit(); // ========================================= } LokNoFailRemoveWordlist( pPart ); } CATCH( CException, e ) { } END_CATCH } //+------------------------------------------------------------------------- // // Member: CResManager::IsMemoryLow, private // // Returns: TRUE if we're in a low memory situation // // History: 10-May-93 AmyA Created from old DoUpdates // 3-May-96 dlee #if 0'ed it out because the memory // load # isn't reliable, and we've // got LOTS of other allocations anyway. // 05-Nov-97 KyleP New approach // //-------------------------------------------------------------------------- BOOL CResManager::IsMemoryLow() { SCODE sc = _xLowRes->IsMemoryLow(); if ( E_NOTIMPL == sc ) sc = _xLowResDefault->IsMemoryLow(); return ( S_OK == sc ); } //+------------------------------------------------------------------------- // // Member: CResManager::IsIoHigh, private // // Returns: TRUE if the system is performing a 'lot' of I/O // // History: 10-Dec-97 KyleP Created // // Notes: This call takes time (~ 5 sec) to complete. // //-------------------------------------------------------------------------- BOOL CResManager::IsIoHigh() { SCODE sc = _xLowRes->IsIoHigh( &_fStopMerge ); if ( E_NOTIMPL == sc ) sc = _xLowResDefault->IsIoHigh( &_fStopMerge ); return ( S_OK == sc ); } //+------------------------------------------------------------------------- // // Member: CResManager::IsBatteryLow, private // // Returns: TRUE if the system is running low on battery power. // // History: 16-Jul-98 KyleP Created // // Notes: By default, even 100% battery (as opposed to A/C) may be // considered low. // //-------------------------------------------------------------------------- BOOL CResManager::IsBatteryLow() { SCODE sc = _xLowRes->IsBatteryLow(); if ( E_NOTIMPL == sc ) sc = _xLowResDefault->IsBatteryLow(); return ( S_OK == sc ); } //+------------------------------------------------------------------------- // // Member: CResManager::IsOnBatteryPower, private // // Returns: TRUE if the system is running on battery power. // // History: 01-Oct-00 dlee Created // //-------------------------------------------------------------------------- BOOL CResManager::IsOnBatteryPower() { SCODE sc = _xLowRes->IsOnBatteryPower(); if ( E_NOTIMPL == sc ) sc = _xLowResDefault->IsOnBatteryPower(); return ( S_OK == sc ); } //IsOnBatteryPower //+------------------------------------------------------------------------- // // Member: CResManager::SampleUserActivity, private // // Returns: - nothing - // // History: 31 Jul 98 AlanW Created // //-------------------------------------------------------------------------- void CResManager::SampleUserActivity() { SCODE sc = _xLowRes->SampleUserActivity(); if ( E_NOTIMPL == sc ) _xLowResDefault->SampleUserActivity(); return; } //+------------------------------------------------------------------------- // // Member: CResManager::IsUserActive, private // // Arguments: [fCheckLongTerm] - TRUE if long-term activity should be checked. // // Returns: TRUE if the interactive user is using the keyboard or mouse // // History: 28 Jul 98 AlanW Created // //-------------------------------------------------------------------------- BOOL CResManager::IsUserActive(BOOL fCheckLongTerm) { SCODE sc = _xLowRes->IsUserActive(fCheckLongTerm); if ( E_NOTIMPL == sc ) sc = _xLowResDefault->IsUserActive(fCheckLongTerm); return ( S_OK == sc ); } //====================== // STATE AND STATISTICS //====================== //+------------------------------------------------------------------------- // // Member: CResManager::CountPendingUpdates, public // // Arguments: [secCount (output)] -- Count of secondary Q Updates // // Returns: Count of pending updates. // // History: 14-Dec-92 KyleP Created // // Notes: If wordlists are being constructed the count may be low. // //-------------------------------------------------------------------------- unsigned CResManager::CountPendingUpdates(unsigned& secCount) { CPriLock lock( _mutex ); unsigned count = 0; CPartIter iter; secCount = 0; for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList)) { count += iter.LokGet()->LokCountUpdates(); secCount += iter.LokGet()->LokCountSecUpdates(); } return(count + secCount); } //+------------------------------------------------------------------------- // // Member: CResManager::UpdateCounters, public // // Returns: Updates performance counters. // // History: 05-Jan-95 BartoszM Created // //-------------------------------------------------------------------------- void CResManager::LokUpdateCounters() { if ( !_isDismounted ) { _prfCounter.Update( CI_PERF_NUM_WORDLIST, _partList.LokWlCount() ); unsigned cSecQDocuments; unsigned cUnfilteredDocs = CountPendingUpdates( cSecQDocuments ); _prfCounter.Update( CI_PERF_FILES_TO_BE_FILTERED, cUnfilteredDocs ); _prfCounter.Update( CI_PERF_DEFERRED_FILTER_FILES, cSecQDocuments ); _prfCounter.Update( CI_PERF_NUM_UNIQUE_KEY, _sKeyList->MaxKeyIdInUse() ); _prfCounter.Update( CI_PERF_DOCUMENTS_FILTERED, _cFilteredDocuments ); ULONG cDocuments; GetClientState( cDocuments ); _prfCounter.Update( CI_PERF_NUM_DOCUMENTS, cDocuments ); } } //==================== // UPDATES AND QUERIES //==================== //+--------------------------------------------------------------------------- // // Member: CResManager::ReserveUpdate, public // // Arguments: [wid] -- Used to confirm hint. // // Returns: Hint to position of reserved slot. // // History: 30-Aug-95 KyleP Created // //---------------------------------------------------------------------------- unsigned CResManager::ReserveUpdate( WORKID wid ) { CPriLock lock ( _pendQueue.GetMutex() ); return _pendQueue.LokPrepare( wid ); } //+--------------------------------------------------------------------------- // // Member: CResManager::UpdateDocument, public // // Arguments: [iHint] -- Positional hint // [wid] -- wid to add/delete // [partid] -- partition containing this wid // [usn] -- USN associated with change // [volumeId] -- Volume Id // [action] -- addition / deletion // // History: 08-Apr-91 BartoszM Created // 08-Oct-93 BartoszM Rewrote to accept single document // 30-Aug-95 KyleP Use reserved slots // //---------------------------------------------------------------------------- SCODE CResManager::UpdateDocument( unsigned iHint, WORKID wid, PARTITIONID partid, USN usn, VOLUMEID volumeId, ULONG action ) { if ( !IsIndexingEnabled() ) return CI_E_FILTERING_DISABLED; Win4Assert ( partid == 1 ); // // Try to avoid lock contention // with filter agent // _pFilterAgent->SlowDown(); CPriLock lock ( _mutex ); BOOL fComplete; { CPriLock lock2( _pendQueue.GetMutex() ); if ( _isBeingEmptied ) // The content index is empty { // // Just get rid of any pending entries in the queue. // if ( !_pendQueue.LokComplete( iHint, wid, usn, volumeId, partid, action ) ) { while ( _pendQueue.LokRemove( wid, usn, volumeId, partid, action ) ) ; } return CI_E_SHUTDOWN; } // // The if clause triggers when this document is the only one // on the queue. // fComplete = _pendQueue.LokComplete( iHint, wid, usn, volumeId, partid, action ); } SCODE sc = S_OK; if ( fComplete ) { CPartition* pPart = _partList.LokGetPartition ( partid ); CChangeTrans xact( *this, pPart ); sc = pPart->LokUpdateDocument ( xact, wid, usn, volumeId, action, 1, 0 ); xact.LokCommit(); _pFilterAgent->LokWakeUp(); } else { BOOL fGotOne = FALSE; CPendQueueTrans pendQueueTrans( _pendQueue ); while ( TRUE ) { BOOL fGotAnother; { CLock lock2( _pendQueue.GetMutex() ); fGotAnother = _pendQueue.LokRemove( wid, usn, volumeId, partid, action ); } if ( fGotAnother ) { fGotOne = TRUE; CPartition* pPart = _partList.LokGetPartition ( partid ); CChangeTrans xact( *this, pPart ); sc = pPart->LokUpdateDocument ( xact, wid, usn, volumeId, action, 1, 0 ); xact.LokCommit(); } else break; } pendQueueTrans.Commit(); if ( fGotOne ) _pFilterAgent->LokWakeUp(); } return sc; } //+--------------------------------------------------------------------------- // // Member: CResManager::FlushUpdates // // Synopsis: Flushes all update notifications to disk // // History: 27-Jun-97 SitaramR Created // //---------------------------------------------------------------------------- void CResManager::FlushUpdates() { CPriLock lock( _mutex ); CPartition * pPart = _partList.LokGetPartition( partidDefault ); Win4Assert( 0 != pPart ); pPart->LokFlushUpdates(); } //+--------------------------------------------------------------------------- // // Member: CResManager::QueryIndexes, public // // Arguments: [cPartitions] -- count of partitions // [aPartID] -- array of partition id's // [freshTest] -- return arg.: fresh test // [cInd] -- return arg: count of indexes // [cPendingUpdates] -- Pending update 'threshold'. If fewer // pending updates, the pending wids are // returned in *pcurPending. // [pcurPending] -- Pending cursors stored here. // [pFlags] -- return arg: status of indexes // // Returns: Array of pointers to indexes // // History: 08-Oct-91 BartoszM Created // // Notes: Called by Query // Indexes and fresh test have ref count increased // Flags may change to indicate status of indexes // //---------------------------------------------------------------------------- CIndex** CResManager::QueryIndexes ( unsigned cPartitions, PARTITIONID aPartID[], CFreshTest** freshTest, unsigned& cInd, ULONG cPendingUpdates, CCurStack * pcurPending, ULONG* pFlags ) { unsigned count = 0; CPartition* pPart; CPriLock lock ( _mutex ); if ( _isCorrupt ) THROW( CException( CI_CORRUPT_DATABASE ) ); if ( _isBeingEmptied ) // Content index is empty or corrupt { cInd = 0; cPartitions = 0; return 0; } for ( unsigned i = 0; i < cPartitions; i++ ) { pPart = _partList.LokGetPartition( aPartID [i] ); count += pPart->LokIndexCount(); } XArray apIndex( count ); #if CIDBG || DBG unsigned j = 0; #endif for ( i = 0; i < cPartitions; i++ ) { pPart = _partList.LokGetPartition(aPartID [i]); #if CIDBG || DBG j += #endif // CIDBG || DBG pPart->LokGetIndexes ( &apIndex[i] ); if ( LokIsScanNeeded() && !_storage.IsReadOnly() ) *pFlags |= CI_NOT_UP_TO_DATE; // // If we're looking for pending updates, get them. Otherwise just // set the out-of-date flag. // unsigned cPending = pPart->LokCountUpdates(); cPending += pPart->LokCountSecUpdates(); if ( cPending != 0 ) { ULONG fFlags = CI_NOT_UP_TO_DATE; // assume non-success if ( cPendingUpdates > 0 ) { Win4Assert( 0 != pcurPending ); if ( cPending <= cPendingUpdates ) { XArray pWid( cPending ); BOOL fSucceeded = pPart->LokGetPendingUpdates( pWid.GetPointer(), cPending ); if ( fSucceeded ) { Win4Assert( cPending > 0 ); XPtr xCursor( new CPendingCursor( pWid, cPending ) ); pcurPending->Push( xCursor.GetPointer() ); xCursor.Acquire(); fFlags = 0; } } } *pFlags |= fFlags; } } // // Also check to see if we are actually in the process of filtering some // documents. // if ( _docList.Count() > 0 ) { if ( _docList.Count() <= cPendingUpdates ) { Win4Assert( 0 != pcurPending ); XArray pWid( _docList.Count() ); ciDebugOut((DEB_FILTERWIDS, "CResManager::QueryIndexes - pending documents: %d %x\n", _docList.Count(), pWid.GetPointer())); _docList.LokGetWids( pWid ); XPtr xCursor( new CPendingCursor( pWid, _docList.Count() ) ); pcurPending->Push( xCursor.GetPointer() ); xCursor.Acquire(); } else *pFlags |= CI_NOT_UP_TO_DATE; } // // Also check documents pending notifcation that may be complete but // out-of-order. // { CPriLock lock( _pendQueue.GetMutex() ); unsigned cwidPending = _pendQueue.LokCountCompleted(); if ( cwidPending > 0 ) { if ( cwidPending <= cPendingUpdates ) { Win4Assert( 0 != pcurPending ); XArray pWid( cwidPending ); _pendQueue.LokGetCompleted( pWid.GetPointer() ); XPtr xCursor( new CPendingCursor( pWid, cwidPending ) ); pcurPending->Push( xCursor.GetPointer() ); xCursor.Acquire(); } else *pFlags |= CI_NOT_UP_TO_DATE; } } // // Finally, check for pending scans. // if ( _isOutOfDate ) *pFlags |= CI_NOT_UP_TO_DATE; Win4Assert ( j == count ); *freshTest = _fresh.LokGetFreshTest(); cInd = count; return apIndex.Acquire(); } //+--------------------------------------------------------------------------- // // Member: CResManager::ReleaseIndexes, public // // Synopsis: Decrements ref counts, deletes if indexes // marked to be deleted. // // Arguments: [cInd] -- count of indexes // [apIndex] -- array of indexes // [freshTest] -- fresh test // // History: 08-Oct-91 BartoszM Created // // Notes: Takes ResMan lock // //---------------------------------------------------------------------------- void CResManager::ReleaseIndexes ( unsigned cInd, CIndex** apIndex, CFreshTest* pFreshTest ) { ciDebugOut (( DEB_ITRACE, "Release Indexes\n" )); CPriLock lock ( _mutex ); for ( unsigned i = 0; i < cInd; i++ ) { if ( apIndex[i] != 0 ) LokReleaseIndex ( apIndex[i] ); } if ( pFreshTest) _fresh.LokReleaseFreshTest (pFreshTest); } //+--------------------------------------------------------------------------- // // Member: CResManager::BackupContentIndexData // // Synopsis: Private method called by the merge thread to backup the // content index data. If successful, it sets the status of // the backup work item to indicate success. Otherwise, it // must be considered to have failed. // // History: 3-18-97 srikants Created // //---------------------------------------------------------------------------- void CResManager::BackupContentIndexData() { Win4Assert( 0 != _pBackupWorkItem ); ciDebugOut(( DEB_WARN, "Starting Backup of Ci Data\n" )); // // Create the backup object. // CPartition *pPartition =_partList.LokGetPartition ( partidDefault ); CBackupCiPersData backup( *_pBackupWorkItem, *this, *pPartition ); // ========================================= { CPriLock lock(_mutex); backup.LokGrabResources(); } // ========================================= backup.BackupIndexes(); // ========================================= { CPriLock lock(_mutex); backup.LokBackupMetaInfo(); ciDebugOut(( DEB_WARN, "Completed Backup of Ci Data\n" )); _pBackupWorkItem->LokSetStatus( STATUS_SUCCESS ); } // ========================================= } //+--------------------------------------------------------------------------- // // Member: CResManager::LokUpdateBackupMergeProgress // // Synopsis: Updates the backup progress during the merge. // // History: 3-20-97 srikants Created // //---------------------------------------------------------------------------- void CResManager::LokUpdateBackupMergeProgress() { Win4Assert( 0 != _pBackupWorkItem ); CCiFrmPerfCounter counter( _adviseStatus.GetPointer(), CI_PERF_MERGE_PROGRESS ); DWORD dwMergeProgress = counter.GetCurrValue(); // // Assuming Merge is half of the work, we will assume half // progress. So make the denominator 200 instead of 100. // if ( _pMerge ) { const cShadowMergePart = 5; if ( _pBackupWorkItem->IsFullSave() ) { if ( _pMerge->GetMergeType() != mtMaster ) { // // The shadow merge preceding a master is approx. 5% // say. // dwMergeProgress = cShadowMergePart; } else { dwMergeProgress = min(dwMergeProgress + cShadowMergePart, 100); } } _pBackupWorkItem->LokUpdateMergeProgress( (ULONG) dwMergeProgress, 100 ); } } //+--------------------------------------------------------------------------- // // Member: CResManager::BackupCIData // // Synopsis: External method to backup content index data. It creates // a backup work item and waits for the backup to complete. // // Arguments: [storage] - Destination storage // [fFull] - [in/out] Set to TRUE if a full merge is // needed on input; On output will be set to TRUE if a full // merge was done. // [progressTracker] - Progress tracker. // // History: 3-18-97 srikants Created // //---------------------------------------------------------------------------- SCODE CResManager::BackupCIData( PStorage & storage, BOOL & fFull, XInterface & xEnumWorkids, PSaveProgressTracker & progressTracker ) { if ( !IsIndexMigrationEnabled() || !IsIndexingEnabled() ) return CI_E_INVALID_STATE; SCODE sc = S_OK; Win4Assert( fFull ); TRY { // =================================================== { CLock lock(_mutex); if ( _isDismounted ) THROW( CException( CI_E_SHUTDOWN ) ); // There can be only one operation going on at a time. if ( 0 != _pBackupWorkItem ) THROW( CException( CI_E_INVALID_STATE ) ); _pBackupWorkItem = new CBackupCiWorkItem( storage, fFull, progressTracker ); _eventMerge.Set(); } // =================================================== ciDebugOut(( DEB_WARN, "Waiting for backup to complete\n" )) ; // // Wait for the work-item to be completed. // while ( TRUE ) { DWORD const dwWaitTime = 1 * 60 * 1000; // 1 minute in millisecs _pBackupWorkItem->WaitForCompletion( dwWaitTime ); //============================================================ { CPriLock lock(_mutex); if ( _pBackupWorkItem->LokIsDone() ) break; // // If there is a merge going on, we must update the progress. // if ( _pBackupWorkItem->LokIsDoingMerge() ) LokUpdateBackupMergeProgress(); _pBackupWorkItem->LokReset(); } //============================================================ } // of while sc= _pBackupWorkItem->GetStatus(); ciDebugOut(( DEB_WARN, "Backup completed with code 0x%X\n", sc )); if ( S_OK == sc ) { fFull = _pBackupWorkItem->IsFullSave(); xEnumWorkids.Set( _pBackupWorkItem->AcquireWorkIdEnum() ); } // ==================================== { CLock lock(_mutex); delete _pBackupWorkItem; _pBackupWorkItem = 0; } // ==================================== } CATCH( CException,e ) { sc = HRESULT_FROM_WIN32( e.GetErrorCode() ); Win4Assert( !"How Did we Come Here" ); } END_CATCH return sc; } //+--------------------------------------------------------------------------- // // Member: CResManager::CiState, public // // Arguments: [state] -- internal state of the CI // // History: 01-Nov-95 DwightKr Created // // //---------------------------------------------------------------------------- #define setState( field, value ) \ if ( state.cbStruct >= ( offsetof( CIF_STATE, field) + \ sizeof( state.field ) ) ) \ { \ state.field = ( value ); \ } #define roundup(a, b) ((a%b) ? (a/b + 1) : (a/b)) NTSTATUS CResManager::CiState(CIF_STATE & state) { CPriLock lock( _mutex ); if ( _isDismounted ) return CI_E_SHUTDOWN; if ( _isCorrupt ) return CI_CORRUPT_DATABASE; setState( cWordList, _partList.LokWlCount() ); setState( cPersistentIndex, _partList.LokIndexCount() - state.cWordList ); // Get the perf counter for the # of queries executed Win4Assert( !_adviseStatus.IsNull() ); long cQ; _adviseStatus->GetPerfCounterValue( CI_PERF_TOTAL_QUERIES, &cQ ); setState( cQueries, cQ ); unsigned cSecQDocuments; ULONG cUnfilteredDocs = CountPendingUpdates(cSecQDocuments) + _docList.Count(); setState( cDocuments, cUnfilteredDocs ); setState( cSecQDocuments, cSecQDocuments ); setState( cFreshTest, LokGetFreshCount() ); CCiFrmPerfCounter counter( _adviseStatus.GetPointer(), CI_PERF_MERGE_PROGRESS ); setState( dwMergeProgress, (ULONG) counter.GetCurrValue() ); setState( cFilteredDocuments, _cFilteredDocuments ); unsigned size = roundup(_partList.LokIndexSize(), ((1024*1024)/CI_PAGE_SIZE)); setState( dwIndexSize, size); setState( cUniqueKeys, _sKeyList->MaxKeyIdInUse() ); state.cbStruct = min( state.cbStruct, sizeof(state) ); DWORD eCiState = _dwFilteringState; // // Set each of the state bits independently. // // // Are we in a shadow or master merge? // if ( 0 != _pMerge ) { switch (_pMerge->GetMergeType() ) { case mtShadow: eCiState |= CIF_STATE_SHADOW_MERGE; break; case mtAnnealing: eCiState |= CIF_STATE_ANNEALING_MERGE; break; case mtIncrBackup: eCiState |= CIF_STATE_INDEX_MIGRATION_MERGE; break; default: eCiState |= CIF_STATE_MASTER_MERGE; break; } } else { CPartition *pPartition = _partList.LokGetPartition ( partidDefault ); if ( pPartition->InMasterMerge() ) eCiState |= CIF_STATE_MASTER_MERGE_PAUSED; } if ( LokIsScanNeeded() ) eCiState |= CIF_STATE_CONTENT_SCAN_REQUIRED; setState( eState, (CIF_STATE_FLAGS) eCiState ); return STATUS_SUCCESS; } //+--------------------------------------------------------------------------- // // Member: CResManager::IndexSize // // Synopsis: Computes the size of the indexes in all partitions // // Arguments: [mbIndex] - On output, will have the size of all indexes // in MBs. // // History: 4-15-96 srikants Created // //---------------------------------------------------------------------------- void CResManager::IndexSize( ULONG & mbIndex ) { CLock lock(_mutex); mbIndex = _partList.LokIndexSize() / ((1024*1024)/CI_PAGE_SIZE); } //+--------------------------------------------------------------------------- // // Member: CResManager::LokCheckWordlistQuotas, public // // Synopsis: Determine if we've reached wordlist capacity. // // Returns: TRUE if we have as much wordlist data in memory as // parameters allow. // // History: 14-Jan-1999 KyleP Created // //---------------------------------------------------------------------------- BOOL CResManager::LokCheckWordlistQuotas() { CPartIter iter; for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList) ) { CPartition* pPart = iter.LokGet(); if ( pPart->LokCheckWordlistMerge() ) return TRUE; } return FALSE; } //+--------------------------------------------------------------------------- // // Member: CResManager::MarkCorruptIndex // // Synopsis: Marks the index as corrupt, disables updates in all partitions // and wakes up the merge thread to notify the framework client // about the corruption. // // History: 1-30-97 srikants Created // //---------------------------------------------------------------------------- NTSTATUS CResManager::MarkCorruptIndex() { CPriLock lock(_mutex); // // Disable updates in all partitions // CPartIter iter; for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList)) { CPartition * pPart = iter.LokGet(); pPart->LokDisableUpdates(); } if ( _state.FLokCorruptionNotified() ) return S_OK; _isCorrupt = TRUE; StopCurrentMerge(); _eventMerge.Set(); return S_OK; } //===================== // KEYINDEX //===================== #ifdef KEYLIST_ENABLED //+--------------------------------------------------------------------------- // // Member: CResManager::_AddRefKeyList, private // // Synopsis: Adds to refcount on the keylist // // History: 07-Jul-94 dlee Created // //---------------------------------------------------------------------------- CKeyList * CResManager::_AddRefKeyList() { CPriLock lock( _mutex ); _sKeyList->Reference(); return _sKeyList.GetPointer(); } //+--------------------------------------------------------------------------- // // Member: CResManager::_ReleaseKeyList, private // // Synopsis: Release a refcount on the keylist // // History: 07-Jul-94 dlee Created // //---------------------------------------------------------------------------- void CResManager::_ReleaseKeyList() { CPriLock lock( _mutex ); _sKeyList->Release(); if ( !_sKeyList->InUse() && _sKeyList->IsZombie() ) { ciDebugOut(( DEB_ITRACE, "Keylist %x unreferenced zombie. Deleting\n", _sKeyList->GetId() )); CKeyList * pKeyList = _sKeyList.Acquire(); pKeyList->Remove(); delete pKeyList; } } #endif // KEYLIST_ENABLED //+--------------------------------------------------------------------------- // // Member: CResManager::KeyToId, public // // Synopsis: Maps from a key to an id. // // Arguments: [pkey] -- pointer to the key to be mapped to ULONG // // Returns: key id - a ULONG // // History: 03-Nov-93 w-Patg Created. // 17-Feb-94 KyleP Initial version // //---------------------------------------------------------------------------- ULONG CResManager::KeyToId( CKey const * pkey ) { #ifdef KEYLIST_ENABLED CKeyList * pKeyList = _AddRefKeyList(); ULONG kid = pKeyList->KeyToId( pkey ); _ReleaseKeyList(); return( kid ); #else return widInvalid; #endif // KEYLIST_ENABLED } //+--------------------------------------------------------------------------- // // Member: CResManager::IdToKey, public // // Synopsis: Maps from an id to a key. // // Arguments: [ulKid] -- key id to be mapped to a key // [rkey] -- reference to the returned key // // Returns: void // // History: 03-Nov-93 w-Patg Created. // 17-Feb-94 KyleP Initial version // //---------------------------------------------------------------------------- void CResManager::IdToKey( ULONG ulKid, CKey & rkey ) { #ifdef KEYLIST_ENABLED CKeyList * pKeyList = _AddRefKeyList(); pKeyList->IdToKey( ulKid, rkey ); _ReleaseKeyList(); #else rkey.FillMin(); #endif // KEYLIST_ENABLED } //+------------------------------------------------------------------------- // // Member: CResManager::ComputeRelevantWords, public // // Synopsis: Computes and returns relevant words for a set of wids // // Arguments: [cRows] -- # of rows to compute and in pwid array // [cRW] -- # of rw per wid // [pwid] -- array of wids to work over // [partid] -- partition // // History: 10-May-94 v-dlee Created // //-------------------------------------------------------------------------- CRWStore * CResManager::ComputeRelevantWords(ULONG cRows,ULONG cRW, WORKID *pwids, PARTITIONID partid) { CRWStore *prws = 0; #ifdef KEYLIST_ENABLED CPriLock lock( _mutex ); CPartition *pPart = LokGetPartition(partid); if (pPart != 0) { CPersIndex *pIndex= pPart->GetCurrentMasterIndex(); if (pIndex != 0) { prws = pIndex->ComputeRelevantWords(cRows,cRW,pwids, _sKeyList.GetPointer() ); } } #endif // KEYLIST_ENABLED return prws; } //ComputeRelevantWords //+------------------------------------------------------------------------- // // Member: CRWStore::RetrieveRelevantWords, public // // Synopsis: Retrieves relevant words already computed // // Arguments: [fAcquire] -- TRUE if ownership is transferred. // [partid] -- partition // // History: 10-May-94 v-dlee Created // //-------------------------------------------------------------------------- CRWStore * CResManager::RetrieveRelevantWords(BOOL fAcquire, PARTITIONID partid) { CRWStore *prws = 0; #ifdef KEYLIST_ENABLED CPriLock lock( _mutex ); CPartition *pPart = LokGetPartition(partid); if (pPart != 0) { prws = pPart->RetrieveRelevantWords(fAcquire); } #endif // KEYLIST_ENABLED return prws; } //RetrieveRelevantWords //================ // PRIVATE METHODS //================ //+------------------------------------------------------------------------- // // Member: CResManager::CompactResources // // History: 05-Jan-95 BartoszM Created // //-------------------------------------------------------------------------- void CResManager::CompactResources() { CPriLock lock( _mutex ); CPartIter iter; for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList)) { iter.LokGet()->LokCompact(); } } //=================================// // MERGES // //=================================// //+--------------------------------------------------------------------------- // // Member: CResManager::DoMerges, private // // History: 08-Apr-91 BartoszM Created // 25-Feb-92 BartoszM Rewrote to use thread // 30-Jun-97 KrishnaN Calling docstore.FlushPropertyStore // before a shadow merge. // 03-Mar-98 KitmanH Don't merge if _storage is read-only // // Notes: Entry point for captive threads // //---------------------------------------------------------------------------- void CResManager::DoMerges() { if ( _storage.IsReadOnly() ) return; CCiFrmPerfCounter pPerIndexCounter( _adviseStatus.GetPointer(), CI_PERF_NUM_PERSISTENT_INDEXES ); CCiFrmPerfCounter pIndexSizeCounter( _adviseStatus.GetPointer(), CI_PERF_INDEX_SIZE ); CCiFrmPerfCounter pPendingCounter( _adviseStatus.GetPointer(), CI_PERF_FILES_TO_BE_FILTERED ); CCiFrmPerfCounter pNumKeysCounter( _adviseStatus.GetPointer(), CI_PERF_NUM_UNIQUE_KEY); CCiFrmPerfCounter pWordListCounter(_adviseStatus.GetPointer(), CI_PERF_NUM_WORDLIST); CCiFrmPerfCounter pDeferredFilesCounter( _adviseStatus.GetPointer(), CI_PERF_DEFERRED_FILTER_FILES ); XPtr xIdle( new CIdleTime() ); // // Allow mount to complete before checking for merges. O/W we will // start a merge before the mount/startup is complete and it might // prevent a system from coming up. // const lowDiskPollTime = 1; // 1 minute DWORD dwWaitTime = _isLowOnDiskSpace ? lowDiskPollTime : _frmwrkParams.GetMaxMergeInterval(); _eventMerge.Wait( dwWaitTime * 1000 * 60 ); BOOL fAnnealing = FALSE; while ( !_fStopMerge ) { BOOL fCorrupt = FALSE; ciDebugOut (( DEB_ITRACE, "\t|Merge Wakeup!\n" )); // // Book keeping chores. // BOOL fBackupCiData = FALSE; BOOL fBackupStarted = FALSE; TRY { // =============================================== CPriLock lock(_mutex); if ( _isCorrupt ) { if ( !_state.FLokCorruptionNotified() ) LokNotifyCorruptionToClient(); } else { // // Use low disk condition plus the current state to decide if // updates should be enabled or not // _isLowOnDiskSpace = LokCheckIfDiskLow( *this, _storage, _isLowOnDiskSpace, _adviseStatus.GetReference() ); if ( _isLowOnDiskSpace ) { if ( _state.LokGetState() != eUpdatesToBeDisabled && _state.LokGetState() != eUpdatesDisabled ) { _state.LokSetState( eUpdatesToBeDisabled ); _state.LokSetUpdateType( eIncremental ); } } else { if ( _state.LokGetState() == eUpdatesDisabled ) _state.LokSetState( eUpdatesToBeEnabled ); } if ( _state.LokGetState() == eUpdatesToBeDisabled ) LokNotifyDisableUpdatesToClient(); else if ( _state.LokGetState() == eUpdatesToBeEnabled ) LokNotifyEnableUpdatesToClient(); } fBackupCiData = 0 != _pBackupWorkItem; // =============================================== } CATCH( CException, e ) { // ignore and try again the next time through the loop. } END_CATCH TRY { if ( !_isCorrupt && _partidToMerge != partidInvalid ) { ciDebugOut (( DEB_ITRACE, "Forced merge in %ld\n", _partidToMerge )); switch ( _mtForceMerge ) { case CI_MASTER_MERGE: MasterMerge ( _partidToMerge ); break; case CI_SHADOW_MERGE: Merge( mtShadow, _partidToMerge ); break; case CI_ANNEALING_MERGE: Merge( mtAnnealing, _partidToMerge ); break; case CI_ANY_MERGE: break; // Do nothing. Let normal mechanism work it out. default: Win4Assert( !"Invalid ForceMerge type" ); } } if ( !_isLowOnDiskSpace && !_isCorrupt ) { // // Refile the documents from the secondary queue to the primary // queue in pull filtering. // // ======================= lock ====================== if ( !_fPushFiltering ) { CLock lock(_mutex); LokRefileSecQueueDocs(); } // ======================= unlock ====================== if ( fBackupCiData ) { BOOL fIsMasterPresent = FALSE; BOOL fIsMMergeInProgress = FALSE; //================================ { CPriLock lock(_mutex); _pBackupWorkItem->LokSetDoingMerge(); fIsMasterPresent = LokIsMasterIndexPresent( partidDefault ); fIsMMergeInProgress = LokIsMasterMergeInProgress( partidDefault ); } //================================ if ( _pBackupWorkItem->IsFullSave() || fIsMMergeInProgress || !fIsMasterPresent ) { _pBackupWorkItem->SetDoingFullSave(); MasterMerge( partidDefault ); } else { Merge( mtIncrBackup, partidDefault ); } fBackupStarted = TRUE; BackupContentIndexData(); } else { if ( fAnnealing ) { fAnnealing = FALSE; CPartIdStack partitionIds; CheckAndDoMerge( mtAnnealing, partitionIds ); while ( partitionIds.Count() > 0 ) Merge( mtAnnealing, partitionIds.Pop() ); } { CPartIdStack partitionIds; CheckAndDoMerge( mtShadow, partitionIds ); while ( partitionIds.Count() > 0 ) { // Call FlushPropertyStore on DocStore to give it // a chance to persist changes before a shadow merge. _docStore->FlushPropertyStore(); Merge( mtShadow, partitionIds.Pop() ); } } { if ( CheckDeleteMerge( _partidToMerge != partidInvalid ) ) { // Call FlushPropertyStore on DocStore to give it // a chance to persist changes before a shadow merge. _docStore->FlushPropertyStore(); Merge( mtDeletes, 1 ); } } BOOL fIsMasterPresent = FALSE; BOOL fIsMMergeInProgress = FALSE; //================================ { CPriLock lock(_mutex); fIsMasterPresent = LokIsMasterIndexPresent( partidDefault ); fIsMMergeInProgress = LokIsMasterMergeInProgress( partidDefault ); } //================================ { CPartIdStack partitionIds; CheckAndDoMerge( mtMaster, partitionIds ); while ( partitionIds.Count() > 0 ) MasterMerge( partitionIds.Pop() ); } } } } CATCH ( CException, e ) { ciDebugOut (( DEB_ERROR, "ResMan::DoMerges -- merge failed with 0x%x\n", e.GetErrorCode() )); if ( IsDiskFull(e.GetErrorCode()) ) { ciDebugOut(( DEB_ERROR, "***** DISK IS FULL *****\n" )); CPriLock lock(_mutex); _isLowOnDiskSpace = LokCheckIfDiskLow( *this, _storage, _isLowOnDiskSpace, _adviseStatus.GetReference() ); } else if ( IsCorrupt( e.GetErrorCode() ) ) { Win4Assert( "!Merge failed, but not for low disk. Why?" ); fCorrupt = TRUE; } // // We have to prevent a master merge from being restarted // immediately. The reason for failure could be something like // out of disk space (typical reason) or log full. In either // case we will only be exacerbating the situation if we // restart immediately. // _eventMerge.Reset(); } END_CATCH // ========================================================== { CPriLock lock( _mutex ); Win4Assert( _partList.LokIndexCount() >= _partList.LokWlCount() ); pPerIndexCounter.Update(_partList.LokIndexCount()-_partList.LokWlCount()); pIndexSizeCounter.Update(_partList.LokIndexSize() / ((1024*1024)/CI_PAGE_SIZE) ); pNumKeysCounter.Update(_sKeyList->MaxKeyIdInUse()); pWordListCounter.Update(_partList.LokWlCount()); _partidToMerge = partidInvalid; if ( _fStopMerge ) { ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" )); break; } if ( !_isCorrupt && ( _state.LokGetState() == eSteady ) && ( 0 != _pFilterAgent ) ) _pFilterAgent->LokWakeUp(); _eventMerge.Reset(); // sleep if ( fBackupStarted || _isCorrupt ) { // // Set that backup to be complete even if it failed. Once // Backup starts, it either succeeds or fails but in either // case the operation is considered done. // if ( 0 != _pBackupWorkItem ) { if ( _isCorrupt ) _pBackupWorkItem->LokSetStatus( CI_CORRUPT_DATABASE ); _pBackupWorkItem->LokSetDone(); } } } // ========================================================== BOOL fSteadyState = TRUE; if ( fCorrupt ) { CPriLock lock(_mutex); _isCorrupt = TRUE; fSteadyState = _state.LokGetState() == eSteady; } unsigned cSecQCount; unsigned cPending = CountPendingUpdates( cSecQCount ); // // Wait for either the merge event to be signaled or the deadman timeout. // dwWaitTime = ( ( _isLowOnDiskSpace ) || ( !fSteadyState ) || ( 0 != cPending ) ) ? lowDiskPollTime : _frmwrkParams.GetMaxMergeInterval(); NTSTATUS Status = _eventMerge.Wait( dwWaitTime * 60 * 1000 ); if ( !_isCorrupt ) { unsigned PercentIdle = xIdle->PercentIdle(); // // Are we idle enough to consider an annealing merge? // if ( STATUS_TIMEOUT == Status && PercentIdle >= _frmwrkParams.GetMinMergeIdleTime() ) { ciDebugOut(( DEB_ITRACE, "Idle time for period: %u percent\n", PercentIdle )); fAnnealing = TRUE; } } if ( _fStopMerge ) { ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" )); break; } cPending = CountPendingUpdates( cSecQCount ); pPendingCounter.Update(cPending); pDeferredFilesCounter.Update(cSecQCount); ciDebugOut (( DEB_ITRACE | DEB_PENDING, "%d updates pending\n", cPending )); } // =========================================== { CPriLock lock2(_mutex); if ( _pBackupWorkItem ) { ciDebugOut(( DEB_WARN, "Forcing the backup to be done\n" )); _pBackupWorkItem->LokSetDone(); } } // =========================================== } //+------------------------------------------------------------------------- // // Member: CResManager::StopMerges, private // // Synopsis: Aborts merge-in-progress (if any) and sets merge flag. // // History: 13-Aug-93 KyleP Created // //-------------------------------------------------------------------------- void CResManager::StopMerges() { { CPriLock lock( _mutex ); _fStopMerge = TRUE; _eventMerge.Set(); // wake up // // If we're doing a merge right now then kill it off. // if ( 0 != _pMerge ) { _pMerge->LokAbort(); } } // // Wait for merge thread to die. // _thrMerge.WaitForDeath(); } //+--------------------------------------------------------------------------- // // Member: CResManager::CheckAndDoMerge, private // // History: 08-Apr-91 BartoszM Created // // Notes: Called by captive thread. // //---------------------------------------------------------------------------- void CResManager::CheckAndDoMerge( MergeType mt, CPartIdStack & partitionIds) { if ( !IsBatteryLow() ) { CPriLock lock( _mutex ); ciDebugOut (( DEB_ITRACE, "check merge\n" )); CPartIter iter; for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList)) { CPartition* pPart = iter.LokGet(); if ( mt == mtMaster ) { __int64 shadowIndexSize = pPart->LokIndexSize(); CPersIndex * pMasterIndex = pPart->GetCurrentMasterIndex(); if ( pMasterIndex ) shadowIndexSize -= pMasterIndex->Size(); shadowIndexSize *= CI_PAGE_SIZE; CMasterMergePolicy mergePolicy( _storage, shadowIndexSize, LokGetFreshCount(), _mergeTime, _frmwrkParams, _adviseStatus.GetReference() ); if ( mergePolicy.IsTimeToMasterMerge() || ( mergePolicy.IsMidNightMergeTime() && !IsOnBatteryPower() ) ) partitionIds.Push( pPart->GetId() ); } else { if ( pPart->LokCheckMerge(mt) || ( IsMemoryLow() && pPart->LokCheckLowMemoryMerge() ) ) { #if CIDBG == 1 if ( !pPart->LokCheckMerge(mt) ) { ciDebugOut(( DEB_WARN, "Merge due to low memory. %u wordlists, %u shadow.\n", pPart->WordListCount(), pPart->LokIndexCount() )); } #endif partitionIds.Push( pPart->GetId() ); } } } } } //+--------------------------------------------------------------------------- // // Member: CResManager::CheckDeleteMerge, private // // Synopsis: Merge check for delete merge // // Arguments: [fForce] -- If TRUE, even 1 pending delete will force merge. // // Returns: TRUE if merge should be performed. // // History: 12-Jun-97 KyleP Created // // Notes: Called by captive thread. // //---------------------------------------------------------------------------- BOOL CResManager::CheckDeleteMerge( BOOL fForce ) { CPriLock lock( _mutex ); ciDebugOut (( DEB_ITRACE, "check delete merge\n" )); // // A 'delete merge' occurs when enough documents have been deleted, and no // documents have been added or modified (which would cause a shadow merge). // return ( ( _fresh.LokDeleteCount() > _frmwrkParams.GetMaxFreshDeletes() ) || ( fForce && _fresh.LokDeleteCount() > 0 ) ); } //+--------------------------------------------------------------------------- // // Member: CResManager::Merge, private // // Synopsis: Perform a merge in partition // // Arguments: [partid] -- partition id // [mt] -- Merge type // // Signals: CExcetpion // // History: 08-Apr-91 BartoszM Created // 13-Nov-91 BartoszM Rewrote using CMerge // //---------------------------------------------------------------------------- void CResManager::Merge( MergeType mt, PARTITIONID partid ) { ciDebugOut (( DEB_ITRACE, "%s", (mtDeletes == mt) ? "Delete merging: Have a quick stretch" : "Shadow merging: Get a cup of coffee\n" )); CMerge Merge( *this, partid, mt ); // // Perform resource acquisition under storage transaction. If // aborted all persistent storage will be deleted by the storage // transaction. // { CPriLock lock ( _mutex ); Win4Assert( _pMerge == 0 ); if (_fStopMerge) { ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" )); return; } Merge.LokGrabResources(); // may throw an exception // // Optimization: Don't merge zero indexes. // if ( 0 != Merge.LokCountOld() ) { _pMerge = &Merge; } else if ( mtDeletes != mt ) { ciDebugOut(( DEB_ITRACE, "Shadow merge of zero indexes. Cancelled.\n" )); return; } } XPtr xNotifyTrans; // Perform the merge under simple transaction // No lock held. { TRY { // // Perform merge, if necessary. // if ( 0 != Merge.LokCountOld() ) { CCiFrmPerfCounter counter( _adviseStatus.GetPointer(), CI_PERF_MERGE_PROGRESS ); Merge.Do( counter ); } // // NotifTrans should be outside resman lock and the resman lock // is acquired in notifTran's destructor // xNotifyTrans.Set( new CNotificationTransaction( this, _xIndexNotifTable.GetPointer() ) ); } CATCH ( CException, e ) { CPriLock lock ( _mutex ); Merge.LokRollBack(); _pMerge = 0; // Really bad errors indicate the index is corrupt. SCODE scE = e.GetErrorCode(); if ( STATUS_INTEGER_DIVIDE_BY_ZERO == scE || STATUS_ACCESS_VIOLATION == scE || STATUS_IN_PAGE_ERROR == scE ) { ciDebugOut(( DEB_ERROR, "Corrupt index, caught 0x%x\n", scE )); _storage.ReportCorruptComponent( L"ShadowMerge" ); THROW( CException( CI_CORRUPT_DATABASE ) ); } RETHROW(); } END_CATCH } // nb: no failures allowed until CMergeTrans is constructed! CDiskIndex* pIndexNew = 0; { //=================== begin notification transaction ========================= { CLock lock( _mutex ); XPtr xFreshTestAtMerge; { _pMerge = 0; CMergeTrans Xact( *this, _storage, Merge ); unsigned cIndOld = Merge.LokCountOld(); INDEXID* aIidOld = Merge.LokGetIidList(); pIndexNew = Merge.LokGetNewIndex(); // // Write the new fresh test persistently to disk. // CPersFresh newFreshLog( _storage, _partList ); CShadowMergeSwapInfo swapInfo; swapInfo._widNewFreshLog = _fresh.LokUpdate( Merge, Xact, newFreshLog, pIndexNew ? pIndexNew->GetId() : iidInvalid, cIndOld, aIidOld, xFreshTestAtMerge ); swapInfo._widOldFreshLog = _storage.GetSpecialItObjectId( itFreshLog ); swapInfo._cIndexOld = cIndOld; swapInfo._aIidOld = aIidOld; _partList.LokSwapIndexes( Xact, partid, pIndexNew, swapInfo ); // // No failures allowed after this point on until the transaction // is committed. // Merge.ReleaseNewIndex(); ciDebugOut (( DEB_ITRACE, "done merging\n" )); Xact.LokCommit(); #if CIDBG == 1 if ( pIndexNew ) pIndexNew->Reference(); #endif //========================================== } ciFAILTEST( STATUS_NO_MEMORY ); CPartition *pPartition =_partList.LokGetPartition ( partid ); // // Delete the wids in the change log that have made it to // the new shadow index created in this merge. The algorithm // needs the latest fresh test, fresh test snapshoted at the // start of shadow merge and the doc list. If the latest fresh // test is the same as the fresh test at merge, then as an // optimization, the fresh test at merge is null. So, in the // optimized case, we use the latest fresh test as the fresh // test at merge. // CFreshTest *pFreshTestLatest = _fresh.LokGetFreshTest(); CFreshTestLock freshTestLock( pFreshTestLatest ); CFreshTest *pFreshTestAtMerge; if ( xFreshTestAtMerge.IsNull() ) pFreshTestAtMerge = pFreshTestLatest; else pFreshTestAtMerge = xFreshTestAtMerge.GetPointer(); //========================================== CChangeTrans xact( *this, pPartition ); pPartition->LokDeleteWIDsInPersistentIndexes( xact, *pFreshTestLatest, *pFreshTestAtMerge, _docList, xNotifyTrans.GetReference() ); xact.Commit(); } xNotifyTrans.Free(); } //=================== end notification transaction ========================= #if CIDBG == 1 if ( pIndexNew ) { pIndexNew->VerifyContents(); { CPriLock lock ( _mutex ); LokReleaseIndex( pIndexNew ); } } #endif _storage.CheckPoint(); } //Merge //+--------------------------------------------------------------------------- // // Member: CResManager::LogMMergeStartFailure // // Synopsis: Writes an eventlog message to indicate a MasterMerge could // not be started. // // Arguments: [fRestart] - Indicates if this is a restarted master merge. // [storage] - Storage reference // [error] - Error code. // [adviseStatus] - reference to ICiCAdviseStatus // // History: 5-27-96 srikants Created // Jan-07-96 mohamedn CDmFwEventItem // //---------------------------------------------------------------------------- void CResManager::LogMMergeStartFailure( BOOL fRestart, PStorage & storage, ULONG error, ICiCAdviseStatus & adviseStatus) { TRY { DWORD dwMsgCode = fRestart ? MSG_CI_MASTER_MERGE_CANT_RESTART : MSG_CI_MASTER_MERGE_CANT_START; CDmFwEventItem Item( EVENTLOG_AUDIT_FAILURE, dwMsgCode, 2, sizeof(error), (void *)&error ); Item.AddArg( storage.GetVolumeName() ); Item.AddArg( error ); Item.ReportEvent( adviseStatus ); } CATCH( CException, e ) { ciDebugOut(( DEB_ERROR, "Error 0x%X in LogMMergeStartFailure\n", e.GetErrorCode() )); } END_CATCH } //+--------------------------------------------------------------------------- // // Member: CResManager::LogMMergePaused // // Synopsis: Writes an eventlog message that master merge was paused. // // Arguments: [storage] - Storage // [error] - Error which caused the merge to be paused. // [adviseStatus] - Advise status to use for notifying about the // event. // // History: 1-24-97 srikants Added header // //---------------------------------------------------------------------------- void CResManager::LogMMergePaused( PStorage & storage, ULONG error, ICiCAdviseStatus &adviseStatus) { if ( IsCorrupt( error ) ) { // don't log "paused" when there is corruption return; } TRY { CDmFwEventItem Item( EVENTLOG_INFORMATION_TYPE, MSG_CI_MASTER_MERGE_ABORTED, 2, sizeof(error), (void *)&error ); Item.AddArg( storage.GetVolumeName() ); Item.AddArg( error ); Item.ReportEvent( adviseStatus ); } CATCH( CException, e ) { ciDebugOut(( DEB_ERROR, "Error 0x%X in LogMMergePaused\n", e.GetErrorCode() )); } END_CATCH } //+--------------------------------------------------------------------------- // // Member: CResManager::MasterMerge, private // // Synopsis: Perform a MasterMerge in partition // // Arguments: [partid] -- partition id // // Signals: CException // // History: 08-Apr-91 BartoszM Created // 13-Nov-91 BartoszM Rewrote using CMerge // 17-Feb-94 KyleP Added keylist merge // Jan-07-96 mohamedn CDmFwEventItem // //---------------------------------------------------------------------------- void CResManager::MasterMerge ( PARTITIONID partid ) { ciDebugOut (( DEB_ITRACE, "\n===MASTER MERGE===\n" )); // // Force a shadow merge if we are not restarting a master merge. This // will force all word lists into persistent indexes. // BOOL fMasterMergePaused = FALSE; // // Before starting the master merge, we have to swap the deleted index // iid. However, if there is a failure before we commit the beginning // of the master merge, we have to roll back this change. The // CDeletedIIDTrans object does this. // CDeletedIIDTrans delIIDTrans( *this ); INDEXID iidDelPreMerge = iidInvalid; { CPriLock lock ( _mutex ); CPartition *pPartition = _partList.LokGetPartition ( partid ); fMasterMergePaused = pPartition->InMasterMerge(); if ( !fMasterMergePaused ) { if ( 0 == LokGetFreshCount() ) { // // There is no benefit in doing a master merge. Just return // return; } iidDelPreMerge = _activeDeletedIndex.GetIndex(); delIIDTrans.LokLogNewDeletedIid( iidDelPreMerge, _activeDeletedIndex.GetNewIndex() ); _activeDeletedIndex.SwapIndex(); } } if ( !fMasterMergePaused ) { // // First do a delete merge to finish merging all outstanding // word-lists into a persistent index. // // A delete merge will force an optimized null merge if there are // outstanding deletes in the fresh test which need to be logged. // This is an important case for Push filtering. Since it doesn't // hurt in the other model, just keep the code identical. // Merge( mtDeletes, partid ); } else { // // This is a restarted master merge. The _activeDeletedIndex contains // the "Post" master merge deleted index. The "New" one will be same // as the "Prev" one. // iidDelPreMerge = _activeDeletedIndex.GetNewIndex(); CDmFwEventItem Item( EVENTLOG_INFORMATION_TYPE, MSG_CI_MASTER_MERGE_RESTARTED, 1 ); Item.AddArg( _storage.GetVolumeName() ); Item.ReportEvent( _adviseStatus.GetReference() ); } CMasterMerge Merge( *this, partid ); TRY { CPriLock lock ( _mutex ); Win4Assert( _pMerge == 0 ); if ( _fStopMerge ) { ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" )); return; } ciFAILTEST( STATUS_NO_MEMORY ); CPartition *pPartition =_partList.LokGetPartition ( partid ); if ( !pPartition->InMasterMerge() ) { Win4Assert( delIIDTrans.IsTransLogged() ); Merge.LokGrabResources( delIIDTrans ); // may throw an exception if ( 0 == Merge.LokCountOld() ) { ciDebugOut(( DEB_ITRACE, "Master merge of 0 indexes cancelled\n")); return; } } else { Merge.LokLoadRestartResources(); } // // The deletionIIDTrans must be either committed or not logged at // all. // Win4Assert( !delIIDTrans.IsRollBackTrans() ); _pMerge = &Merge; } CATCH( CException, e ) { ciDebugOut(( DEB_ERROR, "Error 0x%X while starting a MasterMerge\n", e.GetErrorCode() )); LogMMergeStartFailure( fMasterMergePaused, _storage, e.GetErrorCode(), _adviseStatus.GetReference() ); RETHROW(); } END_CATCH // Perform the merge under simple transaction // No lock held. { TRY { // Perform merge CCiFrmPerfCounter counter( _adviseStatus.GetPointer(), CI_PERF_MERGE_PROGRESS ); Merge.Do( counter ); } CATCH ( CException, e ) { CPriLock lock ( _mutex ); Merge.LokRollBack(); _pMerge = 0; LogMMergePaused( _storage, e.GetErrorCode(), _adviseStatus.GetReference() ); RETHROW(); } END_CATCH } CMasterMergeIndex* pMMergeIndex = 0; { CPriLock lock ( _mutex ); { _pMerge = 0; unsigned cIndOld = Merge.LokCountOld(); INDEXID* aIidOld = Merge.LokGetIidList(); pMMergeIndex = Merge.LokGetMasterMergeIndex(); CMergeTrans MergeTrans( *this, _storage, Merge ); ciFAILTEST( STATUS_NO_MEMORY ); //========================================== CMasterMergeTrans xact( *this, _storage, MergeTrans, pMMergeIndex, Merge ); CPartition *pPart = LokGetPartition(partid); #ifdef KEYLIST_ENABLED pPart->SetRelevantWords(pMMergeIndex->AcquireRelevantWords()); #endif // KEYLIST_ENABLED // // Remove old indexes from fresh list after master merge // CPersFresh newFreshLog( _storage, _partList ); CMasterMergeSwapInfo SwapInfo; SwapInfo._widNewFreshLog = _fresh.LokRemoveIndexes ( MergeTrans, newFreshLog, cIndOld, aIidOld, iidDelPreMerge ); SwapInfo._widOldFreshLog = _storage.GetSpecialItObjectId( itFreshLog ); SwapInfo._cIndexOld = cIndOld; SwapInfo._aIidOld = aIidOld; SwapInfo._partid = partid; CKeyList const * pOldKeyList = _sKeyList.GetPointer(); CKeyList const * pNewKeyList = Merge.LokGetNewKeyList(); _partList.LokSwapIndexes( MergeTrans, partid, pMMergeIndex, SwapInfo, pOldKeyList, pNewKeyList ); // // After this point, we must not resume the master merge even // if there is a soft failure. // Merge.ReleaseNewIndex(); #ifdef KEYLIST_ENABLED // // Get rid of the current KeyList and replace it with the // new KeyList // _storage.SetSpecialItObjectId( itMMKeyList, widInvalid ); CKeyList * pKeyList = _sKeyList.Acquire(); Win4Assert( 0 != pKeyList ); _sKeyList.Set( Merge.LokGetNewKeyList() ); Merge.LokReleaseNewKeyList(); pKeyList->Zombify(); if ( !pKeyList->InUse() ) { pKeyList->Remove(); delete pKeyList; } #else CKeyList * pKeyList = _sKeyList.Acquire(); Win4Assert( 0 != pKeyList ); _sKeyList.Set( Merge.LokGetNewKeyList() ); Merge.LokReleaseNewKeyList(); delete pKeyList; #endif // KEYLIST_ENABLED #if CIDBG == 1 pMMergeIndex->Reference(); #endif xact.LokCommit(); //========================================== ciDebugOut (( DEB_ITRACE, "done merging\n" )); } // // There is no need to compact the change log here because the // participants in the master merge are persistent indexes and // there will be no wids to be deleted from the change log when // persistent indexes are merged into another persistent index. // } #if CIDBG == 1 pMMergeIndex->VerifyContents(); { CPriLock lock ( _mutex ); LokReleaseIndex( pMMergeIndex ); } #endif _storage.CheckPoint(); CDmFwEventItem Item( EVENTLOG_INFORMATION_TYPE, MSG_CI_MASTER_MERGE_COMPLETED, 1 ); Item.AddArg( _storage.GetVolumeName() ); Item.ReportEvent( _adviseStatus.GetReference() ); } //+--------------------------------------------------------------------------- // // Class: CReleaseMMergeIndex // // Purpose: Class to acquire the ownernship of the target and current // master indexes during the destruction of a mastermerge index // and release the target and current indexes. // // History: 9-29-94 srikants Created // //---------------------------------------------------------------------------- class CReleaseMMergeIndex { public: CReleaseMMergeIndex( CResManager & resman, CMasterMergeIndex * mmIndex ); ~CReleaseMMergeIndex(); private: CResManager & _resman; CMasterMergeIndex * _pmmIndex; CPersIndex * _pTargetMaster; CPersIndex * _pCurrentMaster; }; CReleaseMMergeIndex::CReleaseMMergeIndex( CResManager & resman, CMasterMergeIndex * mmIndex ) : _resman(resman), _pmmIndex(mmIndex), _pTargetMaster(0), _pCurrentMaster(0) { Win4Assert( _pmmIndex ); Win4Assert( _pmmIndex->IsZombie() && !_pmmIndex->InUse() ); _pmmIndex->AcquireCurrentAndTarget( &_pCurrentMaster, &_pTargetMaster ); Win4Assert( 0 == _pCurrentMaster || _pCurrentMaster->IsZombie() ); if ( 0 != _pCurrentMaster ) { _pCurrentMaster->Reference(); } Win4Assert( 0 != _pTargetMaster ); _pTargetMaster->Reference(); } CReleaseMMergeIndex::~CReleaseMMergeIndex() { delete _pmmIndex; if ( 0 != _pCurrentMaster ) _resman.LokReleaseIndex( _pCurrentMaster ); _resman.LokReleaseIndex( _pTargetMaster ); } //+--------------------------------------------------------------------------- // // Member: CResManager::LokReleaseIndex, private // // Synopsis: Decrements ref counts, deletes if index // marked to be deleted. // // Arguments: [pIndex] -- index to be released // // History: 08-Oct-91 BartoszM Created // // Notes: ResMan LOCKED // //---------------------------------------------------------------------------- void CResManager::LokReleaseIndex ( CIndex* pIndex ) { pIndex->Release(); if ( pIndex->IsZombie() && !pIndex->InUse() ) { CIndexId iid = pIndex->GetId(); // // If it is not a master index, it has to be removed from the // Partition object. A master index would have been already removed // when merge completed. // // If we are in the Empty path, then we want to force all indexes // to be deleted if their ref-count is 0. // if ( !pIndex->IsMaster() || _isBeingEmptied ) { if ( !_isDismounted ) { TRY { _partList.LokRemoveIndex ( iid ); } CATCH( CException, e ) { ciDebugOut(( DEB_ERROR, "Index 0x%08x could not be deleted from index table\n", iid )); } END_CATCH TRY { // Must impersonate as system since query threads // impersonating as a query user can't delete the // files sometimes deleted when an index is released // after a merge. Impersonation can fail, which will // leak the file on disk. CImpersonateSystem impersonate; if ( iid.IsPersistent() ) pIndex->Remove(); } CATCH( CException, e ) { ciDebugOut(( DEB_ERROR, "Index 0x%08x could not be deleted from disk, %#x\n", iid, e.GetErrorCode() )); } END_CATCH } else { // // Zombie indexes are cleaned up after reboot. // // // After a dismount, we should not have any dirty streams // and releasing a persistent zombie index will update the // index table on disk. That is not allowed. This situation // happens only when queries have not been released and // shutdown is happening. // // ciDebugOut(( DEB_WARN, "Index with iid 0x%X being released AFTER CI dismount\n", iid )); } delete pIndex; } else { // // This must be a master merge index as no other master index // can be a zombie. // Win4Assert( pIndex->IsMasterMergeIndex() ); CReleaseMMergeIndex mmIndexRel( *this, (CMasterMergeIndex *)pIndex ); } } } //+--------------------------------------------------------------------------- // // Function: LokReplaceMasterIndex // // Synopsis: Replaces the CMasterMergeIndex with the CPersIndex incarnation // of the target master index in the index list. Also prepares // the CMasterMergeIndex for removal from the in-memory index // list by zombifying it. // // Arguments: [pMMergeIndex] -- The target index in the CMasterMergeIndex // form. // [pNewMaster] -- The CPersIndex form of the target index. // // History: 6-30-94 srikants Created // //---------------------------------------------------------------------------- void CResManager::LokReplaceMasterIndex ( CMasterMergeIndex* pMMergeIndex ) { Win4Assert( pMMergeIndex->IsMaster() && pMMergeIndex->IsZombie() ); CPersIndex * pNewMaster = pMMergeIndex->GetTargetMaster(); Win4Assert( pNewMaster->IsMaster() && !pNewMaster->IsZombie() ); // // Delete the master merge index from the partition. // CIndexId iid = pMMergeIndex->GetId(); CPartition * pPart = _partList.LokGetPartition( iid.PartId() ); pPart->LokRemoveIndex ( iid ); // // Check if there is a query in progress. If not, we can // go ahead and delete it. If a query is in progress, the // indexsnapshot of the query will delete it. // if ( !pMMergeIndex->InUse() ) { Win4Assert( !pNewMaster->InUse() ); pMMergeIndex->ReleaseTargetAndCurrent(); delete pMMergeIndex; } // // Add the new master index to the partition object. This will // now be the only master accessible to new queries. // pPart->AddIndex( pNewMaster ); } // //+--------------------------------------------------------------------------- // // Function: RollBackDeletedIIDChange // // Synopsis: Rolls back the changed made to the deleted iid during the // beginning of a master merge. // // Arguments: [xact] - The transaction object which is rolling back the // change to the deleted iid. // // History: 7-17-95 srikants Created // // Notes: Consider the following sequence of events: // // 1. Pre-MasterMerge preparations : Change Deleleted IID to iidDeleted2 from // iidDeleted1. // // 2. Start a shadow merge. // // 3. Commit the beginning of a new MasterMerge // // 4. Complete the master merge ..... // // If there is a failure between steps 2 and 3, we just revert the deleted // IID back to iidDeleted2. Even if some entries were created in the fresh // list between steps 2 & 3 of the form, WID->iidDeleted2, it is okay. They // will get deleted from the fresh list on the second MasterMerge. // //---------------------------------------------------------------------------- void CResManager::RollBackDeletedIIDChange( CDeletedIIDTrans & xact ) { CPriLock lock(_mutex); Win4Assert( _activeDeletedIndex.GetIndex() == xact.GetNewDelIID() ); Win4Assert( _activeDeletedIndex.GetNewIndex() == xact.GetOldDelIID() ); _activeDeletedIndex.SwapIndex(); } //======================== // FILTER RELATED METHODS //======================== //+--------------------------------------------------------------------------- // // Function: LokGetFilterDocs // // Synopsis: Gets documents from the change log for filtering. _docList // will be filled with the documents. // // Arguments: [cMaxDocs] -- [in] Maximum number of docs to retrieve // [cDocs] -- [out] Number of documents retrieved // [state] -- [in/out] State of the document retrieval // progress. // // Returns: Number of documents in the docBuffer. // // History: 6-16-94 srikants Separated for making kernel mode call // Asynchronous. // // Notes: This function has two goals: // // 1. Select documents from different partitions in a round // robin fashion. // //---------------------------------------------------------------------------- BOOL CResManager::LokGetFilterDocs ( ULONG cMaxDocs, ULONG & cDocs, CGetFilterDocsState & state ) { ciDebugOut(( DEB_ITRACE, "# CResManager::LokGetFilterDocs.\n" )); Win4Assert( state.GetTriesLeft() > 0 && state.GetTriesLeft() <=2 ); cDocs = 0; Win4Assert ( !_isBeingEmptied ); unsigned maxDocs = min( cMaxDocs, CI_MAX_DOCS_IN_WORDLIST); if ( _partIter.LokAtEnd() ) return FALSE; // There are no valid partitions // we are sure that the partition is valid CPartition* pPart = _partIter.LokGet(); Win4Assert ( pPart != 0 ); // // Don't bother looking for more documents to filter if there // are too many wordlists in memory -- wait until they are // merged. // if ( pPart->WordListCount() < _frmwrkParams.GetMaxWordlists() ) { CChangeTrans xact( *this, pPart ); pPart->LokQueryPendingUpdates ( xact, maxDocs, _docList ); xact.LokCommit(); } else { ciDebugOut(( DEB_ITRACE, "Too many wordlists. Waking merge\n" )); _eventMerge.Set(); } _docList.SetPartId ( pPart->GetId() ); //================================== // // If there is a failure after this point, we have to re-add // the documents to the change log. // cDocs = _docList.Count(); for (unsigned i = 0; i < cDocs; i++) { if (_docList.Status(i) != DELETED && _docList.Status(i) != WL_NULL) break; } BOOL fRetry = FALSE; if ( i == cDocs && i != 0 ) { CIndexTrans xact( *this ); _fresh.LokDeleteDocuments ( xact, _docList, _activeDeletedIndex.GetIndex() ); xact.LokCommit(); cDocs = 0; _docList.LokClear(); state.Reset(); // // If we are doing a bunch of deletions in a "delnode" case, // we should see if the number of changed documents has exceeded // the max fresh test count and wake up the merge thread. Note // that we are not starting a merge here - the merge thread will // deal with the policy issues. // if ( LokGetFreshCount() > _frmwrkParams.GetMaxFreshCount() || CheckDeleteMerge( FALSE ) ) { _eventMerge.Set(); } LokUpdateCounters(); } _partIter.LokAdvance( _partList ); if ( _partIter.LokAtEnd() ) { _partIter.LokInit( _partList ); state.DecrementTries(); fRetry = (0 == cDocs) && (state.GetTriesLeft() > 0); } else { fRetry = ( 0 != cDocs ); } Win4Assert( cDocs == _docList.Count() ); return fRetry; } //+--------------------------------------------------------------------------- // // Member: CResManager::FillDocBuffer, public // // Synopsis: Fills buffer with strings for paths and properties of docs // to be filtered // // Arguments: [docBuffer] -- (in, out) buffer to be filled with document // names. // [cb] -- (in, out) count of bytes in docBuffer // [cDocs] -- (out) count of # documents to filter // [state] -- [in/out] State of the document retrieval // progress. // // History: 08-Apr-1991 BartoszM Created // 24-Nov-1992 KyleP Retry pending documents // 22-Mar-1993 AmyA Split into several functions // 22-Jan-1997 SrikantS Changed doc. names to bytes for frame // work. // 18-May-2000 KitmanH Changed to push retried docs to filter // one by one // //---------------------------------------------------------------------------- void CResManager::FillDocBuffer( BYTE * docBuffer, ULONG & cb, ULONG & cDocs, CGetFilterDocsState & state ) { Win4Assert( docBuffer != 0 ); // ========================================= { CPriLock lock(_mutex); _docList.LokSortOnWid(); } // ========================================== unsigned cDocuments = _docList.Count(); // // Format of the data // 4 Bytes remaining files count, 2 Bytes number of files. Following // that is (2 Bytes of Name Length in bytes, Name)* // if ( _fPushFiltering ) { // // We can't take an exception while growing the aborted wids // array while doing exception processing in the filter manager. // So, pre-allocate the array now to insure we won't run out of // room later. Now is a good time to die; later isn't. // CPriLock lock( _mutex ); _aAbortedWids.LokReserve( cDocuments ); // // In push filtering there are no refiles, and hence the buffer // cannot overflow. Check that the buffer can fit 16 documents. // A document name is the serialized form of the wid, i.e. it's // 4 bytes long. The buffer size is 4K. // Win4Assert( cb > 4 + 2 + 16 * (2 + 4) ); } const cbLenField = sizeof(ULONG); Win4Assert( cb >= cbLenField ); Win4Assert( sizeof(ULONG) == 4 ); if ( cb < cbLenField ) THROW( CException( STATUS_INVALID_PARAMETER ) ); // // Fill the number of documents left in the changelog in the first // 4 bytes of the buffer. // // ========================================================= { CPriLock lock( _mutex ); ULONG partid = 1; CPartition * pPart = _partList.LokGetPartition( partid ); ULONG cDocsLeft = pPart->LokCountUpdates(); RtlCopyMemory( docBuffer, &cDocsLeft, sizeof(ULONG) ); } // ========================================================= RtlZeroMemory( docBuffer + cbLenField, sizeof USHORT ); const cbUSHORT = sizeof USHORT; BYTE * pCount = docBuffer + cbLenField; *pCount = 0; BYTE * pBuf = pCount + cbUSHORT; BYTE * pEnd = docBuffer + cb; cDocs = 0; // copy path/property pairs into buffer until we run out of either // buffer space or pairs. We stop filling the buffer if we encounter a // document that has a retry count > 1, we will fill the docBuffer with // these docs one at a time ciDebugOut((DEB_FILTERWIDS | DEB_ITRACE, "CResManager::FillDocBuffer...")); unsigned cbExtraSpaceNeeded = 0; for (unsigned cSrcDocs = 0; cSrcDocs < cDocuments; cSrcDocs++) { // // Leave the first 2 bytes for the length field // unsigned cbPath = (unsigned)(pEnd - pBuf); if ( cbPath <= cbUSHORT ) { // We would definitely need more space than a USHORT, but // we don't know how much right now. But since the filter // increases space by a PAGE, I guess we should do fine. cbExtraSpaceNeeded = cbUSHORT - cbPath + 1; break; } if (_docList.Status(cSrcDocs) == DELETED) { cbPath = 0; ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, " %dX", _docList.Wid(cSrcDocs) )); } else { if ( _docList.Retries(cSrcDocs) > 1 ) { ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "************FillDocBuffer: Retry count is %d. Wid is %d\n", _docList.Retries(cSrcDocs), _docList.Wid(cSrcDocs) )); ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "cDocs is %d\n", cDocs )); if ( cDocs > 0 ) { // // There is at least one document in the docBuffer. Stop // now so we can file the doc(s) being retried one by one. // break; } else { // // Add the currenct doc. Stop if the current add is successful // // // Make room for the length field at the beginning. // cbPath -= cbUSHORT; ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, " %d", _docList.Wid(cSrcDocs) )); unsigned cbPathIn = cbPath; // // If this is not the first try, then attempt to get a more // accurate path. We may be suffering from NTBUG: 270566 // // changes cbPath. if ( !WorkidToDocName( _docList.Wid(cSrcDocs), pBuf+cbUSHORT, cbPath, TRUE ) ) { if ( cbPathIn < cbPath) { cbExtraSpaceNeeded = cbPath - cbPathIn; } break; } // cbPath = 0 if object has been deleted (not added to docBuffer) if ( cbPath == 0 ) { _docList.SetStatus( cSrcDocs, DELETED ); ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "X" )); } else { ClearNonStoragePropertiesForWid( _docList.Wid(cSrcDocs) ); cSrcDocs++; cDocs++; USHORT uscb = (USHORT) cbPath; RtlCopyMemory( pBuf, &uscb, cbUSHORT ); pBuf += (cbPath+cbUSHORT); break; } } } else // not retries { ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "FillDocBuffer: Not retry for wid %d\n", _docList.Wid(cSrcDocs) )); // // Make room for the length field at the beginning. // cbPath -= cbUSHORT; ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, " %d", _docList.Wid(cSrcDocs) )); unsigned cbPathIn = cbPath; // changes cbPath. if ( !WorkidToDocName( _docList.Wid(cSrcDocs), pBuf+cbUSHORT, cbPath, FALSE ) ) { if ( cbPathIn < cbPath) { cbExtraSpaceNeeded = cbPath - cbPathIn; } break; } // cbPath = 0 if object has been deleted if ( cbPath == 0 ) { _docList.SetStatus( cSrcDocs, DELETED ); ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "X" )); } else { cDocs++; ClearNonStoragePropertiesForWid( _docList.Wid(cSrcDocs) ); } } } USHORT uscb = (USHORT) cbPath; RtlCopyMemory( pBuf, &uscb, cbUSHORT ); pBuf += (cbPath+cbUSHORT); } ciDebugOut (( DEB_FILTERWIDS | DEB_NOCOMPNAME, "\n" )); // // Fill in the count of the number of documents. // USHORT usFileCount = (USHORT) cSrcDocs; RtlCopyMemory( pCount, &usFileCount, cbUSHORT ); if ( cSrcDocs < cDocuments ) { if ( 0 == cSrcDocs && cbExtraSpaceNeeded > 0 ) { // We need a larger buffer cb += cbExtraSpaceNeeded; } // // No refiling in push filtering // Win4Assert( !_fPushFiltering ); ciDebugOut (( DEB_ITRACE, "Putting back documents\n" )); CDocList docListExtra; docListExtra.SetPartId( _docList.PartId() ); for ( unsigned i = 0, j = cSrcDocs; j < cDocuments; i++, j++ ) { ciDebugOut(( DEB_ITRACE | DEB_NOCOMPNAME, " %d", _docList.Wid(j) )); ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "Refiling Wid %d\n", _docList.Wid(j) )); docListExtra.Set( i, _docList.Wid(j), _docList.Usn(j), _docList.VolumeId(j) ); } ciDebugOut (( DEB_ITRACE | DEB_NOCOMPNAME, "\n" )); CPriLock lock2(_mutex); docListExtra.LokSetCount(i); _docList.LokSetCount( cSrcDocs ); ReFileCommit ( docListExtra ); } // // If we were unable to convert any of the wids to paths, then all of // them have been deleted. // if ( 0 == cDocs ) { ciDebugOut (( DEB_ITRACE, "All docs were deleted\n" )); CPriLock lock3(_mutex); _docList.LokClear(); state.Reset(); } else { cb = (ULONG)(pBuf - docBuffer); } } //+--------------------------------------------------------------------------- // // Function: NoFailReFile // // Synopsis: re-adds the documents to the change log witout committing // // History: 6-16-94 srikants Separated from FilterReady for making // kernel mode call Asynchronous. // //---------------------------------------------------------------------------- BOOL CResManager::NoFailReFile ( CDocList& docList ) { // // No refiling in push filtering // Win4Assert( !_fPushFiltering ); CPriLock lock ( _mutex ); if ( _isBeingEmptied ) return FALSE; LokNoFailReFileChanges( docList ); docList.LokClear(); return TRUE; } //+--------------------------------------------------------------------------- // // Function: ReFileCommit // // Synopsis: re-adds the documents to the change log and commits // them to the change list. // // History: 6-16-94 srikants Separated from FilterReady for making // kernel mode call Asynchronous. // //---------------------------------------------------------------------------- BOOL CResManager::ReFileCommit ( CDocList& docList ) { // // No refiling in push filtering // Win4Assert( !_fPushFiltering ); CPriLock lock ( _mutex ); if ( _isBeingEmptied ) return FALSE; LokNoFailReFileChanges( docList ); docList.LokClear(); // // The docqueue can handle only one set of refiled documents. // we MUST complete processing of the // refiled documents. // PARTITIONID partId = 1; CPartition * pPart = LokGetPartition( partId ); { // ========================================= CChangeTrans xact( *this, pPart ); pPart->LokAppendRefiledDocs( xact ); xact.LokCommit(); // ========================================= } return TRUE; } //+--------------------------------------------------------------------------- // // Function: LokNoFailReFileChanges // // Synopsis: Refiles the updates with the change log. // // Arguments: [docList] -- contains the documents to be added back to the // change log. // // History: 5-16-94 srikants Modified to deal with failures during // append of docs to change log. // // Notes: This CANNOT throw // //---------------------------------------------------------------------------- void CResManager::LokNoFailReFileChanges( CDocList& docList ) { // // No refiling in push filtering // Win4Assert( !_fPushFiltering ); Win4Assert ( docList.PartId() == 1 ); CPartition* pPart = _partList.LokGetPartition ( docList.PartId() ); pPart->LokRefileDocs( docList ); _pFilterAgent->LokWakeUp (); } //+------------------------------------------------------------------------- // // Member: CResManager::LokReFileDocument, public // // Returns: Put a document back on the quueue to be refiltered // // History: 05-Jan-95 BartoszM Created // //-------------------------------------------------------------------------- void CResManager::LokReFileDocument ( PARTITIONID partid, WORKID wid, USN usn, VOLUMEID volumeId, ULONG retries, ULONG secQRetries ) { // // No refiling in push filtering // Win4Assert( !_fPushFiltering ); CPartition* pPart = _partList.LokGetPartition(partid); //========================================== CChangeTrans xact( *this, pPart ); // // A usn of 0 is used for all refiled documents // pPart->LokUpdateDocument (xact, wid, 0, volumeId, CI_UPDATE_OBJ, retries, secQRetries); xact.LokCommit(); //========================================== } //+--------------------------------------------------------------------------- // // Member: CResManager::LokAddToSecQueue // // Synopsis: Adds the given document to the secondary change log of the // partition. // // Arguments: [partid] - PartitionId // [wid] - WorkId of the document // [volumeId] - Volume id // [cSecQRetries] - Sec Q Retry Count // // History: 5-09-96 srikants Created // //---------------------------------------------------------------------------- void CResManager::LokAddToSecQueue( PARTITIONID partid, WORKID wid, VOLUMEID volumeId, ULONG cSecQRetries ) { // // No refiling in push filtering // Win4Assert( !_fPushFiltering ); CPartition* pPart = _partList.LokGetPartition(partid); //========================================== CChangeTrans xact( *this, pPart ); pPart->LokAddToSecQueue (xact, wid, volumeId, cSecQRetries); xact.LokCommit(); //========================================== } //+--------------------------------------------------------------------------- // // Member: CResManager::LokRefileSecQueueDocs // // Synopsis: Transfers the documents from the secondary queue to the primary // change queue. // // History: 5-09-96 srikants Created // //---------------------------------------------------------------------------- void CResManager::LokRefileSecQueueDocs() { // // No refiling in push filtering // Win4Assert( !_fPushFiltering ); CPartIter iter; for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList)) { CPartition * pPart = iter.LokGet(); //========================================== CChangeTrans xact( *this, pPart ); pPart->LokRefileSecQueue( xact ); xact.LokCommit(); //========================================== } } //+------------------------------------------------------------------------- // // Member: CResManager::LokTransferWordlist, public // // Returns: Transfer a wordlist from the filter manager // to the content index // // History: 05-Jan-95 BartoszM Created // //-------------------------------------------------------------------------- BOOL CResManager::LokTransferWordlist ( PWordList& pWordList ) { BOOL fSuccess = TRUE; PARTITIONID partid = _docList.PartId(); CPartition* pPart = _partList.LokGetPartition(partid); INDEXID iid = pWordList->GetId(); TRY { //========================================== CIndexTrans xact( *this ); // // LokDone can fail while trying to add unfinished docs to the // change log. In that case we must disable further USN updates. // We should commit the change transaction immediately after the // LokDone succeeds. // CChangeTrans changeXact( *this, pPart ); pPart->LokDone ( changeXact, iid, _docList ); ciFAILTEST( STATUS_NO_MEMORY ); changeXact.LokCommit(); _fresh.LokAddIndex ( xact, iid, _activeDeletedIndex.GetIndex(), _docList, *pWordList ); ciDebugOut (( DEB_ITRACE, "CFilterManager::FilterDone " "Success: transfering wordlist\n" )); pWordList->Done(); // // This may be an empty wordlist: all documents were deleted or in error. // If so, just get rid of the wordlist. // if ( pWordList->IsEmpty() ) { #if CIDBG == 1 unsigned cShouldHaveData = 0; unsigned cDocuments = _docList.Count(); for ( unsigned iDoc = 0; iDoc < cDocuments; iDoc++ ) { switch ( _docList.Status(iDoc) ) { case TOO_MANY_RETRIES: case SUCCESS: case TOO_MANY_BLOCKS_TO_FILTER: cShouldHaveData++; break; default: break; } } if ( cShouldHaveData != 0 ) { for ( unsigned iDoc = 0; iDoc < cDocuments; iDoc++ ) { ciDebugOut(( DEB_ERROR, "WID: 0x%x STATUS: %d\n", _docList.Wid(iDoc), _docList.Status(iDoc) )); } } Win4Assert( cShouldHaveData == 0 ); #endif pWordList.Delete(); } else pWordList.Transfer ( pPart ); xact.LokCommit(); } CATCH ( CException, e ) { ciDebugOut(( DEB_ERROR, "Exception 0x%x caught commiting wordlist.\n" "delete wordlist and clear doclist\n", e.GetErrorCode() )); fSuccess = FALSE; } END_CATCH _eventMerge.Set(); // wake up return fSuccess; } //+--------------------------------------------------------------------------- // // Function: LokNoFailRemoveWordlist // // Synopsis: Removes the biggest wordlist from memory and adds the // documents back to the change log. // // Requires: _docList.Count() == 0 when this method is called. This // relies on the fact that there are no other outstanding // wordlists being constructed. // // History: 10-20-94 srikants Added the header section and modified // to deal with the change in refiling // documents. // //---------------------------------------------------------------------------- void CResManager::LokNoFailRemoveWordlist( CPartition * pPart ) { // // In push filtering, we don't delete completed wordlist to simplify // the logic of aborting the notification transaction on the client // if ( _isBeingEmptied || _fPushFiltering ) return; // // Look for biggest wordlist // unsigned cInd; CIndex ** apIndex = pPart->LokQueryMergeIndexes( cInd, mtWordlist ); XPtrST xIndex( apIndex ); unsigned size = 0; CWordList * pWordList = 0; for ( unsigned i = 0; i < cInd; i++ ) { apIndex[i]->Release(); if ( apIndex[i]->Size() > size ) { Win4Assert( apIndex[i]->IsWordlist() ); pWordList = (CWordList *)apIndex[i]; size = pWordList->Size(); } } if ( pWordList ) { // // Reschedule contents for later filtering. // INDEXID iid = pWordList->GetId(); ciDebugOut(( DEB_ITRACE, "Removing wordlist %x to reduce resource consumption.\n", iid )); CDocList _docList; _docList.SetPartId ( pPart->GetId() ); pWordList->GetDocuments( _docList ); #if CIDBG == 1 for ( unsigned cWid = 0; cWid < _docList.Count(); cWid++ ) { ciDebugOut(( DEB_ITRACE, "Reschedule filtering of wid %d\n", _docList.Wid( cWid ) )); } #endif // CIDBG == 1 LokNoFailReFileChanges ( _docList ); // // Remove it // ciDebugOut(( DEB_WARN, "Deleting wordlist with USN (%x, %x).\n", lltoHighPart(pWordList->Usn()), (ULONG) pWordList->Usn() )); pPart->LokRemoveIndex( iid ); if ( !pWordList->InUse() ) { _partList.LokRemoveIndex ( iid ); delete pWordList; } else { pWordList->Zombify(); } } } //+------------------------------------------------------------------------- // // Member: CResManager::LokMakeWordlistId, public // // Returns: Makes unique wordlist id // // History: 05-Jan-95 BartoszM Created // //-------------------------------------------------------------------------- INDEXID CResManager::LokMakeWordlistId (PARTITIONID partid) { CPartition* pPart = _partList.LokGetPartition(partid); if ( pPart == 0 ) return iidInvalid; // Get unique index id INDEXID iid = pPart->LokMakeWlstId (); ciDebugOut((DEB_FILTERWIDS, "CResManager::LokMakeWorkListId - new wordlist iid: %x\n", iid)); return iid; } //+------------------------------------------------------------------------- // // Member: CResManager::ReportFilterFailure, public // // Synopsis: Write to event log about unsuccessful filtering // // Arguments: [wid] -- workid of the file that couldn't be filtered // // History: 05-Jan-95 BartoszM Created // //-------------------------------------------------------------------------- void CResManager::ReportFilterFailure (WORKID wid) { PROPVARIANT var; var.vt = VT_UI4; var.ulVal = wid; _adviseStatus->NotifyStatus( CI_NOTIFY_FILTERING_FAILURE, 1, &var ); } //========= // SCANNING //========= //+--------------------------------------------------------------------------- // // Member: CResManager::SetUpdateLost // // Synopsis: Sets that an update got lost and wakes up the merge thread // to notify the client about it. // // Arguments: [partId] - Partition id // // History: 1-24-97 srikants Created // //---------------------------------------------------------------------------- void CResManager::SetUpdatesLost( PARTITIONID partId ) { CPriLock lock(_mutex); if ( _state.LokGetState() != eUpdatesToBeDisabled ) { _state.LokSetState( eUpdatesToBeDisabled ); _state.LokSetUpdateType( eIncremental ); StopCurrentMerge(); _eventMerge.Set(); } } //+--------------------------------------------------------------------------- // // Member: CResManager::LokCleanupForFullCiScan, public // // History: 10-Nov-94 DwightKr Created // // Notes: This routine performs the disk I/O and other operations // which does the cleanup after a corruption is detected. // // This routine is similar to the Empty() method, except // that we do not delete the change log object or the // freshlog object, and the index list must contain // the change log and fresh log after the cleanup is complete. // //---------------------------------------------------------------------------- void CResManager::LokCleanupForFullCiScan() { if ( 0 != _pMerge ) // Stop in progress merges. _pMerge->LokAbort(); // // Delete everything from the index table here first. // _idxTab->LokEmpty(); // // If anything fails after this point, chkdsk /f or autochk will // release the disk storage associated with the leaked objects // since they are no longer part of the persistent index list. // // // For each partition, zombify all indexes, and re-add the change // log to the index table on disk. // CPartIter iter; for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList)) { CPartition* pPart = iter.LokGet(); Win4Assert( pPart != NULL ); pPart->LokDisableUpdates(); // // Zombify all indexes in this partition, and delete them if // their ref-count is 0. // unsigned cIndex; CIndex ** aIndex = pPart->LokZombify( cIndex ); ReleaseIndexes( cIndex, aIndex, NULL ); delete [] aIndex; WORKID widMMergeLog; WORKID widDummy; pPart->GetMMergeObjectIds( widMMergeLog, widDummy, widDummy ); if ( widMMergeLog != widInvalid) _storage.RemoveObject( widMMergeLog ); // // Empty the fresh test and the fresh log, and add it to the // index table on disk. // pPart->LokEmpty(); // Release change list/log storage WORKID oidChangeLog = pPart->GetChangeLogObjectId(); Win4Assert( oidChangeLog != widInvalid ); _partList.LokAddIt( oidChangeLog, itChangeLog, pPart->GetId() ); } // // Empty the fresh test, fresh log, and add the fresh log to // the index table on disk. // _fresh.LokInit(); // Release fresh test/log storage WORKID oidFreshLog = _storage.GetSpecialItObjectId(itFreshLog); Win4Assert( oidFreshLog != widInvalid ); _partList.LokAddIt( oidFreshLog, itFreshLog, partidFresh1 ); #ifdef KEYLIST_ENABLED WORKID widKeyList = _storage.GetSpecialItObjectId( itMMKeyList ); #else WORKID widKeyList = widInvalid; #endif // KEYLIST_ENABLED WORKID widPhrLat = _storage.GetSpecialItObjectId( itPhraseLat ); if ( widKeyList != widInvalid) { _storage.RemoveObject( widKeyList ); _storage.SetSpecialItObjectId( itMMKeyList, widInvalid ); } if ( widPhrLat != widInvalid) { _storage.RemoveObject( widPhrLat ); _storage.SetSpecialItObjectId( itPhraseLat, widInvalid ); } _isBeingEmptied = FALSE; // Allow new queries } // =========================================================== // Content Index Frame Work Interfacing. // =========================================================== //+--------------------------------------------------------------------------- // // Member: CResManager::WorkidToDocName // // Synopsis: Converts the given workid to path and fill it in the buffer // given. // // Arguments: [wid] -- Workid to convert. // [pbBuf] -- Destination buffer; // [cb] -- On input the total number of bytes that can be // put. On output, the actual number of bytes copied. // For a deleted file, the cb will be 0. // [fAccurate] -- TRUE --> Attempt to get a more accurate // workid-to-path translation. // // Returns: TRUE if successfully converted; FALSE means the buffer was // not big enough. // // History: 12-04-96 srikants Created // // Notes: WorkidToDocName is called outside the resman lock. It is a // potentially long operation and taking resman lock is not // advisable. Also, creation of a CWorkidToDocName object once // for each wid->docName conversion is expensive. So, we create // one in resman constructor and protect it with a separate // lock. // //---------------------------------------------------------------------------- BOOL CResManager::WorkidToDocName( WORKID wid, BYTE * pbBuf, unsigned & cb, BOOL fAccurate ) { CLock lock(_workidToDocNameMutex); ULONG cbDocName; BYTE const * pbDocName = _xWorkidToDocName->GetDocName( wid, cbDocName, fAccurate ); BOOL fStatus = TRUE; if ( pbDocName ) { ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "WorkidToDocName: wid is %d and DocName is %ws\n", wid, pbDocName )); if ( cbDocName > cb ) fStatus = FALSE; else RtlCopyMemory( pbBuf, pbDocName, cbDocName ); cb = cbDocName; } else { cb = 0; } return fStatus; } // WorkidToDocName //+--------------------------------------------------------------------------- // // Member: CResManager::GetClientState // // Synopsis: Gets the client specific status information like total // number of documents. // // Arguments: [cDocuments] - [out] Total # of documents // // History: 12-04-96 srikants Created // //---------------------------------------------------------------------------- void CResManager::GetClientState( ULONG & cDocuments ) { CI_CLIENT_STATUS status; SCODE sc = _docStore->GetClientStatus( &status ); if ( S_OK == sc ) { cDocuments = status.cDocuments; } else { cDocuments = 0; } } //+--------------------------------------------------------------------------- // // Member: CResManager::LokNotifyCorruptionToClient // // Synopsis: Informs the client that the catalog is corrupt // // History: 12-04-96 srikants Created // //---------------------------------------------------------------------------- void CResManager::LokNotifyCorruptionToClient() { Win4Assert( _isCorrupt ); DisableUpdates( partidDefault ); SCODE sc = _docStore->DisableUpdates( FALSE, CI_CORRUPT_INDEX ); if ( SUCCEEDED( sc ) ) _state.LokSetCorruptionNotified(); // // Disabled following assert so that the general NT 5 user is not // bothered by this assert. // //Win4Assert( ! "We must empty the Content Index" ); // Empty(); } //+--------------------------------------------------------------------------- // // Member: CResManager::LokNotifyDisableUpdatesToClient // // Synopsis: Informs the client that updates should be disabled // // History: 07-05-97 SitaramR Created // //---------------------------------------------------------------------------- void CResManager::LokNotifyDisableUpdatesToClient() { Win4Assert( _state.LokGetState() == eUpdatesToBeDisabled ); DisableUpdates( partidDefault ); BOOL fIncremental = _state.LokGetUpdateType() == eIncremental; SCODE sc = _docStore->DisableUpdates( fIncremental, CI_LOST_UPDATE ); if ( SUCCEEDED(sc) ) { ciDebugOut(( DEB_WARN, "Notified disabled updates to DocStore\n" )); _state.LokSetState( eUpdatesDisabled ); } } //+--------------------------------------------------------------------------- // // Member: CResManager::LokNotifyEnableUpdatesToClient // // Synopsis: Informs the client that updates should be enabled // // History: 07-05-97 SitaramR Created // //---------------------------------------------------------------------------- void CResManager::LokNotifyEnableUpdatesToClient() { Win4Assert( _state.LokGetState() == eUpdatesToBeEnabled ); EnableUpdates( partidDefault ); SCODE sc = _docStore->EnableUpdates(); if ( SUCCEEDED(sc) ) { ciDebugOut(( DEB_WARN, "Notified enable updates to DocStore\n" )); _state.LokSetState( eSteady ); } } //+--------------------------------------------------------------------------- // // Member: CResManager::StoreValue // // Synopsis: Stores the given value in the document store // // Arguments: [wid] - WorkId of the document // [ps] - The FULLPROPSPEC of the property // [var] - Value of the property // [fInSchema] - Returns TRUE if the property is in the schema // // Returns: SCODE result // // History: 12-18-96 srikants Created // //---------------------------------------------------------------------------- SCODE CResManager::StoreValue( WORKID wid, CFullPropSpec const & ps, CStorageVariant const & var, BOOL & fInSchema ) { FULLPROPSPEC const * fps = ps.CastToStruct(); PROPVARIANT const * pVar = ConvertToPropVariant( &var ); // Default to TRUE since the return code isn't checked! fInSchema = TRUE; SCODE sc = _propStore->StoreProperty( wid, fps, pVar ); if ( CI_E_PROPERTY_NOT_CACHED == sc ) fInSchema = FALSE; else if ( FAILED( sc ) ) return sc; return S_OK; } BOOL CResManager::StoreSecurity( WORKID wid, PSECURITY_DESCRIPTOR pSD, ULONG cbSD ) { return S_OK == _docStore->StoreSecurity( wid, (BYTE *) pSD, cbSD ); } //+--------------------------------------------------------------------------- // // Member: CResManager::FPSToPROPID, public // // Synopsis: Converts FULLPROPSPEC property to PROPID // // Arguments: [fps] -- FULLPROPSPEC representation of property // [pid] -- PROPID written here on success // // Returns: S_OK on success // // History: 29-Dec-1997 KyleP Created. // //---------------------------------------------------------------------------- SCODE CResManager::FPSToPROPID( CFullPropSpec const & fps, PROPID & pid ) { return _mapper->PropertyToPropid( fps.CastToStruct(), TRUE, &pid ); } // // Flush notify support // //+--------------------------------------------------------------------------- // // Member: CResManager::NotifyFlush // // Synopsis: Notifies the client that changes got flushed and gives the // flush information. // // Arguments: [ft] - FileTime of the last successful flush. // [cEntries] - Number of entries in the aInfo // [aInfo] - Array of USN information for the flush. // // History: 1-27-97 srikants Created // // Note: We should not call directly into framework client because // it violates resman->client lock hierarchy. We should not call // into client with resman lock held. // //---------------------------------------------------------------------------- void CResManager::NotifyFlush( FILETIME const & ft, ULONG cEntries, USN_FLUSH_INFO ** ppInfo ) { if ( !_fPushFiltering ) { // // Notification of flushes are sent in pull filtering only. // For push filtering, the ICiCIndexNotificationStatus // interface is used to notify clients. // CPriLock lock(_mutex); CChangesFlushNotifyItem * pItem = new CChangesFlushNotifyItem( ft, cEntries, ppInfo ); _flushList.Queue( pItem ); if (! _fFlushWorkerActive) { CFwFlushNotify * pWorkItem = new CFwFlushNotify( _workMan, *this ); _workMan.AddToWorkList( pWorkItem ); pWorkItem->AddToWorkQueue(); pWorkItem->Release(); } } } //NotifyFlush //+--------------------------------------------------------------------------- // // Member: CResManager::ProcessFlushNotifies // // Synopsis: Processes the queued flush notifies in a FIFO order. This // method is called only by the asynchronous worker thread // responsible for notifying the client about flushes. // // History: 1-27-97 srikants Created // //---------------------------------------------------------------------------- void CResManager::ProcessFlushNotifies() { // // Only one worker thread allowed to operate on the list at a time. // CLock lock (_mtxChangeFlush ); CRevertBoolValue RevertFlushActive( _fFlushWorkerActive, TRUE ); while ( TRUE ) { CChangesFlushNotifyItem * pItem = 0; // // Remove the first flush item from the list under resman lock. // //=========================================== { CPriLock lock(_mutex); pItem = _flushList.GetFirst(); if ( pItem ) _flushList.Pop(); if (_flushList.IsEmpty()) RevertFlushActive.Revert(); } //=========================================== if ( pItem ) { SCODE sc = _docStore->CheckPointChangesFlushed( pItem->GetFlushTime(), pItem->GetCount(), pItem->GetFlushInfo() ); if ( FAILED(sc) ) { // // We don't have to requeue this item. The client's restart // work will just increase. // ciDebugOut(( DEB_ERROR, "CheckPointChangesFlushed failed. Error (0x%X)\n", sc )); } pItem->Close(); delete pItem; } else { break; } } // While (TRUE) } //ProcessFlushNotifies //+--------------------------------------------------------------------------- // // Member: CResManager::_LokDeleteFlushWorkItems // // Synopsis: Deletes the flush notifies in the queue. This is called // during shutdown (dismount) to clean up any asynchronous // worker threads referring to resman. // // History: 1-27-97 srikants Created // //---------------------------------------------------------------------------- void CResManager::LokDeleteFlushWorkItems() { for ( CChangesFlushNotifyItem * pItem = _flushList.Pop(); 0 != pItem; pItem = _flushList.Pop() ) { pItem->Close(); delete pItem; } } //LokDeleteFlushWorkItems //+--------------------------------------------------------------------------- // // Member: CResManager::NoFailAbortWidsInDocList // // Synopsis: Used in push filtering to abort all wids in the current // doclist // // History: 24-Feb-97 SitaramR Created // //---------------------------------------------------------------------------- void CResManager::NoFailAbortWidsInDocList() { CPriLock lock ( _mutex ); if ( _isBeingEmptied ) return ; for ( unsigned i=0; i<_docList.Count(); i++ ) { WORKID wid = _docList.Wid(i); USN usn = _docList.Usn(i); // // Abort the client transaction // _xIndexNotifTable->AbortWid( wid, usn ); // // Keep track of aborted wids for use during // changlog shrink from front. // _aAbortedWids.NoFailLokAddWid( wid, usn ); } _docList.LokClear(); } //NoFailAbortWidsInDocList //+--------------------------------------------------------------------------- // // Member: CResManager::NoFailAbortWid // // Synopsis: Used in push filtering to abort a wid // // Arguments: [wid] -- Workid // [usn] -- Usn // // History: 24-Feb-97 SitaramR Created // //---------------------------------------------------------------------------- void CResManager::NoFailAbortWid( WORKID wid, USN usn ) { CPriLock lock ( _mutex ); if ( _isBeingEmptied ) return ; // // Abort the client transaction // _xIndexNotifTable->AbortWid( wid, usn ); // // Keep track of aborted wid for use during // changlog shrink from front. // _aAbortedWids.NoFailLokAddWid( wid, usn ); } //NoFailAbortWid //+--------------------------------------------------------------------------- // // Member: CResManager::ClearNonStoragePropertiesForWid // // Synopsis: Clear non-storage properties from the property storage for wid // // Arguments: [wid] -- Workid // // History: 10-Oct-2000 KitmanH Created // //---------------------------------------------------------------------------- SCODE CResManager::ClearNonStoragePropertiesForWid( WORKID wid ) { return _propStore->ClearNonStoragePropertiesForWid( wid ); } //==================== // MASTER MERGE POLICY //==================== //+--------------------------------------------------------------------------- // // Member: CMasterMergePolicy::IsTimeToMasterMerge, public // // Synopsis: Determines if the system should start a master merge // immediately, and not wait until the master merge time. // // History: 4-Aug-94 DwightKr Created // // Notes: If new tests are added simply add them here. // //---------------------------------------------------------------------------- BOOL CMasterMergePolicy::IsTimeToMasterMerge() const { return IsMaxShadowIndexSize() || IsMaxFreshListCount() || IsMinDiskFreeForceMerge(); } //+--------------------------------------------------------------------------- // // Member: CMasterMergePolicy::IsMinDiskFreeForceMerge, private // // Synopsis: If the amount of free disk space is lower than a threshold, // then we should master merge, since it is likely that disk // space will be released. // // History: 4-Aug-94 DwightKr Created // Jan-07-96 mohamedn CDmFwEventItem // //---------------------------------------------------------------------------- BOOL CMasterMergePolicy::IsMinDiskFreeForceMerge() const { // // If we've exceeded the MIN_DISKFREE_FORCE_MERGE space used on // the system, and a master merge has the potential to make some // useful progress, then it's time to force a master merge. The // values of MIN_DISKFREE_FORCE_MERGE and MAX_SHADOW_FREESPACE // are expressed as a percentage. // __int64 diskTotal, diskRemaining; _storage.GetDiskSpace( diskTotal, diskRemaining ); if ( diskRemaining < minDiskFreeForMerge ) return FALSE; if (diskRemaining*100/diskTotal < _frmwrkParams.GetMinDiskFreeForceMerge() && _shadowIndexSize*100/diskRemaining > _frmwrkParams.GetMaxShadowFreeForceMerge() ) { CDmFwEventItem item( EVENTLOG_INFORMATION_TYPE, MSG_CI_MASTER_MERGE_STARTED_TOTAL_DISK_SPACE_USED, 2); item.AddArg( _storage.GetVolumeName() ); item.AddArg( _frmwrkParams.GetMinDiskFreeForceMerge() ); item.ReportEvent(_adviseStatus); ciDebugOut(( DEB_ITRACE | DEB_PENDING, "Master merge, reason: Low on disk space\n" )); return TRUE; } return FALSE; } //+--------------------------------------------------------------------------- // // Member: CMasterMergePolicy::IsMaxFreshListCount, private // // Synopsis: If the number of entries in the fresh list is great than // a threshold, we should master merge, and free up memory // occupied by the fresh hash. // // History: 4-Aug-94 DwightKr Created // Jan-07-96 mohamedn CDmFwEventItem // //---------------------------------------------------------------------------- BOOL CMasterMergePolicy::IsMaxFreshListCount() const { if ( _freshCount > _frmwrkParams.GetMaxFreshCount() ) { CDmFwEventItem item( EVENTLOG_INFORMATION_TYPE, MSG_CI_MASTER_MERGE_STARTED_FRESH_LIST_COUNT, 2); item.AddArg( _storage.GetVolumeName() ); item.AddArg( _frmwrkParams.GetMaxFreshCount() ); item.ReportEvent(_adviseStatus); ciDebugOut(( DEB_ITRACE | DEB_PENDING, "Master merge, reason: More than %d entries in freshhash\n", _frmwrkParams.GetMaxFreshCount() )); return TRUE; } return FALSE; } //+--------------------------------------------------------------------------- // // Member: CMasterMergePolicy::IsMaxShadowIndexSize, private // // Synopsis: If the combined size of all shadow indexes is greater than // a threahold, we should master merge, and free up disk space. // The content index should not take up more than a given // percentage of disk space. // // History: 4-Aug-94 DwightKr Created // Jan-07-96 mohamedn CDmFwEventItem // //---------------------------------------------------------------------------- BOOL CMasterMergePolicy::IsMaxShadowIndexSize() const { __int64 diskTotal, diskRemaining; _storage.GetDiskSpace( diskTotal, diskRemaining ); if ( diskRemaining < minDiskFreeForMerge ) return FALSE; if ( _shadowIndexSize*100/diskTotal > _frmwrkParams.GetMaxShadowIndexSize() ) { CDmFwEventItem item( EVENTLOG_INFORMATION_TYPE, MSG_CI_MASTER_MERGE_STARTED_SHADOW_INDEX_SIZE, 2); item.AddArg( _storage.GetVolumeName() ); item.AddArg( _frmwrkParams.GetMaxShadowIndexSize() ); item.ReportEvent(_adviseStatus); ciDebugOut(( DEB_ITRACE | DEB_PENDING, "Master merge, reason: Combined shadow indexes > %d%% of disk\n", _frmwrkParams.GetMaxShadowIndexSize() )); return TRUE; } return FALSE; } //+--------------------------------------------------------------------------- // // Method: CMasterMergePolicy::ComputeMidNightMergeTime // // Synopsis: Function to compute the time (in hundreds of nanoseconds // from midnight Dec 31, 1969) for the next master merge based on // the value given from registry. // // Arguments: [mergeTime] - The time read from the registry control params. // It's an offset in minutes after midnight. // // Returns: LONG - time for next merge // // History: 19-Feb-1996 srikants Moved the code from svcfilt.cxx // 17-Nov-1999 KyleP Stop using CRT // // Notes: If the time is before January 1, 1970 it will return -1. // //---------------------------------------------------------------------------- ULONGLONG CMasterMergePolicy::ComputeMidNightMergeTime( LONG mergeTime ) { Win4Assert( mergeTime < 24 * 60 ); mergeTime *= 60; // // What time is it here? // SYSTEMTIME stLocal; GetLocalTime( &stLocal ); LONG lNow = stLocal.wHour*60*60 + stLocal.wMinute*60 + stLocal.wSecond; // // Have we passed the daily merge time? // LONGLONG llHNSTilMerge; // Hundred-Nano-Second-Til-Merge LONGLONG const llHNSOneDay = 24*60*60*10000000i64; if ( lNow < mergeTime ) llHNSTilMerge = mergeTime * 10000000i64; else llHNSTilMerge = mergeTime * 10000000i64 + llHNSOneDay; // // Create the *local* filetime for the next merge // stLocal.wHour = 0; stLocal.wMinute = 0; stLocal.wSecond = 0; stLocal.wMilliseconds = 1; FILETIME ftLocal; if ( !SystemTimeToFileTime( &stLocal, &ftLocal ) ) return -1; ((ULARGE_INTEGER *)&ftLocal)->QuadPart += llHNSTilMerge; // // Now, adjust to UTC and return // FILETIME ft; if ( !LocalFileTimeToFileTime( &ftLocal, &ft ) ) return -1; #if CIDBG == 1 FILETIME ft2; GetSystemTimeAsFileTime( &ft2 ); Win4Assert( ((ULARGE_INTEGER *)&ft2)->QuadPart < ((ULARGE_INTEGER *)&ft)->QuadPart ); #endif // CIDBG return ((ULARGE_INTEGER *)&ft)->QuadPart; } //ComputeMidNightMergeTime //+--------------------------------------------------------------------------- // // Member: CMasterMergePolicy::IsMidNightMergeTime, public // // Synopsis: Determines if it's been 24 hours since the last merge time // // History: 4-Aug-1994 DwightKr Created // 17-Nov-1999 KyleP Stop using CRT // //---------------------------------------------------------------------------- BOOL CMasterMergePolicy::IsMidNightMergeTime() { FILETIME ft; GetSystemTimeAsFileTime( &ft ); if ( ((ULARGE_INTEGER *)&ft)->QuadPart > _mergeTime ) { ULONG mergeTime = _frmwrkParams.GetMasterMergeTime(); ULONGLONG oldMergeTime = _mergeTime; _mergeTime = ComputeMidNightMergeTime( mergeTime ); // // If it's been more than 30 minutes since the master merge time, don't // do it. This can happen if we were doing a master merge as part of // indexing documents or if you turn on a machine in the morning and it // was off at the master merge time. // LONGLONG const llHNSHalfHour = 30*60*10000000i64; if ( ((ULARGE_INTEGER *)&ft)->QuadPart > ( oldMergeTime + llHNSHalfHour ) ) return FALSE; return TRUE; } return FALSE; } //IsMidNightMergeTime