//+--------------------------------------------------------------------------- // // Microsoft Windows // Copyright (C) Microsoft Corporation, 1991 - 2000, // // File: MERGE.CXX // // Contents: Merge object // // Classes: CMerge // // History: 13-Nov-91 BartoszM Created // //---------------------------------------------------------------------------- #include #pragma hdrstop #include #include #include "merge.hxx" #include "resman.hxx" #include "partn.hxx" #include "mindex.hxx" #include "mmerglog.hxx" #include "indxact.hxx" const unsigned TWO_MEGABYTES = 0x200000; class CStartMergeTrans : public CTransaction { public: CStartMergeTrans( CMerge & merge ) : _merge(merge) { } ~CStartMergeTrans() { if ( GetStatus() != CTransaction::XActCommit) { _merge.LokRollBack(); } } private: CMerge & _merge; }; //+--------------------------------------------------------------------------- // // Class: XWid // // Purpose: Smart Pointer for a workID, destroys it if not acquired // // History: 11-Apr-95 DwightKr Created // //---------------------------------------------------------------------------- class XWid { public: XWid(WORKID wid, PStorage & storage) : _wid(wid), _storage(storage) { } ~XWid() { if (widInvalid != _wid) _storage.RemoveObject( _wid ); } WORKID Acquire() { WORKID wid=_wid; _wid=widInvalid; return wid; } private: WORKID _wid; PStorage & _storage; }; //+--------------------------------------------------------------------------- // // Member: CMerge::CMerge, public // // Synopsis: Initializes resources to 0 // // Arguments: [resman] -- resource manager // [partid] -- partition id // [mt] -- merge type // // History: 13-Nov-91 BartoszM Created. // //---------------------------------------------------------------------------- CMerge::CMerge ( CResManager& resman, PARTITIONID partid, MergeType mt ) : _partid(partid), _resman(resman), _pPart(0), _iidNew(iidInvalid), _widNewIndex(widInvalid), _mt(mt), _indSnap(resman), _pIndexNew(0), _aIidOld(0), _cIidOld(0) { } //+--------------------------------------------------------------------------- // // Function: LokSetup // // Synopsis: Common setup for shadow merge as well as master merge. // Creates the index snapshot for merge and a wid for the // new index. // // History: 8-30-94 srikants Moved from LokGrabResources as part of // separating CMerge from CMasterMerge. // //---------------------------------------------------------------------------- void CMerge::LokSetup( BOOL fIsMasterMerge ) { Win4Assert( mtAnnealing == _mt || mtShadow == _mt || mtMaster == _mt || mtIncrBackup == _mt || mtDeletes == _mt ); // in case this merge was forced _resman.LokClearForceMerge(); // Get the partition _pPart = _resman.LokGetPartition(_partid); // Create unique persisten index id _iidNew = _pPart->LokMakePersId(); if ( _iidNew == iidInvalid ) { ciDebugOut (( DEB_ITRACE, "Out of persistent index id's\n" )); // No real problem. We'll retry merging later once a query frees // up some indexes. THROW ( CException ( CI_OUT_OF_INDEX_IDS )); } ciFAILTEST( STATUS_NO_MEMORY ); // Initialize source indexes _indSnap.LokInit( *_pPart, _mt ); if (_indSnap.Count() == 0) { return; } USN usnMin = 0x100000000i64; // allocate and initialize array of index id's _cIidOld = _indSnap.Count(); _aIidOld = new INDEXID [_cIidOld]; for ( unsigned i = 0; i < _cIidOld; i++ ) { _aIidOld[i] = _indSnap.GetId(i); if ( _indSnap.GetUsn(i) < usnMin) { usnMin = _indSnap.GetUsn(i); } } // create empty persistent index PStorage::EDefaultStrmType strmType = fIsMasterMerge ? PStorage::eSparseIndex : PStorage::eNonSparseIndex; _widNewIndex = _resman._storage.CreateObjectId( _iidNew, strmType ); } //LokSetup //+--------------------------------------------------------------------------- // // Member: CMerge::LokGrabResources, public // // Synopsis: Initializes all the resources needed to merge // // History: 13-Nov-91 BartoszM Created. // // Notes: ResMan LOCKED // //---------------------------------------------------------------------------- void CMerge::LokGrabResources( ) { Win4Assert( mtAnnealing == _mt || mtShadow == _mt || mtIncrBackup == _mt || mtDeletes == _mt ); // ================================================= CStartMergeTrans xact( *this ); LokSetup( FALSE ); if (_indSnap.Count() == 0) { _pPart->FreeIndexId( _iidNew ); return; } // a piece of heuristics unsigned size = _indSnap.TotalSizeInPages(); ciFAILTEST(STATUS_NO_MEMORY); Win4Assert( 0 == _pIndexNew ); _pIndexNew = new CPersIndex( _resman._storage, _widNewIndex, _iidNew, size, CDiskIndex::eShadow ); ciFAILTEST(STATUS_NO_MEMORY); xact.Commit(); // ================================================= } //LokGrabResources //+--------------------------------------------------------------------------- // // Member: CMerge::~CMerge, public // // Synopsis: Frees resources no longer needed // (whether merge was successful or not) // // History: 13-Nov-91 BartoszM Created. // // Notes: ResMan NOT LOCKED // //---------------------------------------------------------------------------- CMerge::~CMerge() { Win4Assert( 0 == _pIndexNew ); delete _aIidOld; } //+--------------------------------------------------------------------------- // // Member: CMerge::LokRollBack, public // // Synopsis: Puts back old indexes into partition, removes // the new index. Deletes the new index from storage, // Frees the new index id. // // Arguments: [swapped] -- number of old indexes removed from partition // // History: 13-Nov-91 BartoszM Created. // // Notes: ResMan LOCKED // //---------------------------------------------------------------------------- void CMerge::LokRollBack( unsigned swapped ) { ciDebugOut (( DEB_ITRACE, "Merge::RollBack\n" )); if ( swapped != 0 ) { for ( unsigned i=0; i < swapped; i++ ) _pPart->AddIndex ( _indSnap.Get(i) ); if ( swapped == _indSnap.Count() ) _pPart->LokRemoveIndex ( _iidNew ); } if ( 0 != _pIndexNew ) { // Delete from storage _pIndexNew->Remove(); delete _pIndexNew; _pIndexNew = 0; } else if ( widInvalid != _widNewIndex ) { // // This can happen if the new index creation fails after // the wid has been allocated. The wid must be deleted. // _resman._storage.RemoveObject( _widNewIndex ); } if ( _iidNew != iidInvalid ) _pPart->FreeIndexId ( _iidNew ); } //+--------------------------------------------------------------------------- // // Member: CMerge::LokZombify, public // // Synopsis: Deletes old indexes (they are still in use // until the destructor of Merge frees them) // // History: 13-Nov-91 BartoszM Created. // // Notes: ResMan LOCKED // //---------------------------------------------------------------------------- void CMerge::LokZombify() { for ( unsigned i = 0; i < _indSnap.Count(); i++ ) { CIndex* pIndex = _indSnap.Get(i); if ( pIndex != 0 ) { pIndex->Zombify(); } } } //+--------------------------------------------------------------------------- // // Member: CMerge::Do, public // // Synopsis: Do the merge // // Arguments: [mergeProgress] reference to location where % merge complete // can be stored to update perfmon counters // // Signals: CException // // History: 13-Nov-91 BartoszM Created. // // Notes: ResMan NOT LOCKED // //---------------------------------------------------------------------------- void CMerge::Do(CCiFrmPerfCounter & mergeProgress) { _pIndexNew->Merge ( _indSnap, *_pPart, mergeProgress, FALSE // no relevant words computation ); } //+--------------------------------------------------------------------------- // // Member: CMasterMerge::LokStoreRestartResources, private // // Synopsis: Persistently stores resources needed to restart a master merge. // // History: 04-Apr-94 DwightKr Created. // 23-Aug-94 SrikantS Moved from CMerge. // // Notes: ResMan LOCKED // //---------------------------------------------------------------------------- void CMasterMerge::LokStoreRestartResources( CDeletedIIDTrans & delIIDTrans ) { { PRcovStorageObj *pPersMMergeLog = _resman._storage.QueryMMergeLog(_widMasterLog); SRcovStorageObj PersMMergeLog( pPersMMergeLog); // // Inside kernel, we are guaranteed that a new object has no data in // it. In user space, we may be using an object that was not deleted // before due to a failure. // PersMMergeLog->InitHeader(_resman.GetStorageVersion()); // reset the header contents CNewMMergeLog newMMergeLog(*pPersMMergeLog); unsigned cIndOld = LokCountOld(); INDEXID* aIidOld = LokGetIidList(); for (unsigned i=0; iMaxWorkId() ); newMMergeLog.Commit(); newMMergeLog.DoCommit(); } // // Add new index & master log to index list // // CIndexRecord record; record._objectId = _widNewIndex; record._iid = _iidNew; record._type = itNewMaster; record._maxWorkId = 0; // // The index snap shot belongs to the Partition object after the // merge information has been committed in the index table. // Since a memory allocation could fail, we must allocate the // necessary memory before committing the start of master merge. // CIndexSnapshot * pMergeIndSnap = new CIndexSnapshot( _resman ); SIndexSnapshot sMergeIndSnap( pMergeIndSnap ); sMergeIndSnap->LokTakeIndexes( _indSnap ); ciFAILTEST( STATUS_NO_MEMORY ); // // Commit the beginning of a new master merge in the index table. // _resman._idxTab->AddMMergeObjects( _pPart->GetId(), record, _widMasterLog, _widKeyList, delIIDTrans.GetOldDelIID(), delIIDTrans.GetNewDelIID() ); // // Commit the starting of a new master merge by storing the // wids and iids in-memory. // delIIDTrans.Commit(); _resman._storage.SetSpecialItObjectId( itMMKeyList, _widKeyList ); _pPart->SetMMergeObjectIds( _widMasterLog, _widNewIndex, _widCurrentMaster ); _pPart->RegisterId( _iidNew ); _pPart->SetNewMasterIid( _iidNew ); // // Mark all the indexes participating in the master merge in memory // to prevent them from being used for another shadow merge. // unsigned cInd; CIndex** apIndex = sMergeIndSnap->LokGetIndexes( cInd ); for ( unsigned i = 0; i < cInd; i++ ) { apIndex[i]->SetInMasterMerge(); } // // Transfer the ownership of the merge index snapshot to the partition // object. The ownership will be taken over by the MasterMergeIndex // when it is created. Once the beginning of a master merge is // committed, the index snapshot MUST survive until it completes. // Until a CMasterMergeIndex is created, the partition object will be // the owner of this index snapshot. // _pPart->TakeMMergeIndSnap( sMergeIndSnap.Acquire() ); } //+--------------------------------------------------------------------------- // // Member: CMerge::LokLoadRestartResources // // Synopsis: Loads restart resources needed to restart a master merge. // // History: 04-Apr-94 DwightKr Created. // 23-Aug-94 SrikantS Moved from CMerge. // // Notes: ResMan LOCKED // //---------------------------------------------------------------------------- void CMasterMerge::LokLoadRestartResources() { // in case this merge was forced _resman.LokClearForceMerge(); _pPart = _resman.LokGetPartition(_partid); _mt = mtMaster; _fSoftAbort = TRUE; // if there is a failure, only a soft abort // must be done. // ================================================= CStartMergeTrans xact( *this ); #ifdef KEYLIST_ENABLED _widKeyList = _resman._storage.GetSpecialItObjectId( itMMKeyList ); Win4Assert( widInvalid != _widKeyList ); #endif // !KEYLIST_ENABLED _pPart->GetMMergeObjectIds( _widMasterLog, _widNewIndex, _widCurrentMaster ); // // Restore the list of index participating in this merge into the indSnap // object. // { PRcovStorageObj *pLog = _resman._storage.QueryMMergeLog(_widMasterLog); SRcovStorageObj SLog(pLog); // // Since this is a restarted master merge, there is no need to // get the merge indexes. Just get a fresh test. // _indSnap.LokInitFreshTest(); { // // Create the key list which was being built along with the index. // XPtr xMMergeLog( new CMMergeLog( *pLog ) ); // // We should use the minimum key of key list and index split // keys as the split key for the key list. // CKeyBuf * pKeyLstSplitKey = new CKeyBuf(); SKeyBuf sKeyLstSplitKey(pKeyLstSplitKey); BitOffset beginBitOff; BitOffset endBitOff; CKeyBuf * pIdxSplitKey = new CKeyBuf(); SKeyBuf sIdxSplitKey(pIdxSplitKey); xMMergeLog->GetKeyListSplitKeyInfo( *pKeyLstSplitKey, beginBitOff, endBitOff ); xMMergeLog->GetIdxSplitKeyInfo( *pIdxSplitKey, beginBitOff, endBitOff ); if ( pIdxSplitKey->IsMinKey() || pKeyLstSplitKey->Compare( *pIdxSplitKey ) > 0 ) { *pKeyLstSplitKey = *pIdxSplitKey; } INDEXID iidNew = _resman._sKeyList->GetNextIid(); #ifdef KEYLIST_ENABLED _widKeyList = iidNew; _pNewKeyList = new CWKeyList( _resman._storage, _widKeyList, iidNew, _resman._sKeyList.GetPointer(), *pKeyLstSplitKey, xMMergeLog->GetKeyListWidMax() ); #else // KEYLIST_ENABLED _pNewKeyList = new CWKeyList( 0, iidNew ); #endif // !KEYLIST_ENABLED } } // // Create or get the target master index. // CMasterMergeIndex * pIndexNew = LokCreateOrFindNewMaster(); Win4Assert( 0 == _indSnap.Count() ); CIndexSnapshot & indSnap = pIndexNew->LokGetIndSnap(); Win4Assert( 0 != &indSnap ); // // allocate and initialize array of index id's // _cIidOld = indSnap.Count(); _aIidOld = new INDEXID [_cIidOld]; for ( unsigned i = 0; i < _cIidOld; i++ ) { _aIidOld[i] = indSnap.GetId(i); } xact.Commit(); // ================================================= } //+--------------------------------------------------------------------------- // // Member: CMasterMerge::LokGrabResources, public // // Synopsis: Initializes all the resources needed to merge // // History: 13-Nov-91 BartoszM Created. // 23-Aug-94 SrikantS ReWrote for CMasterMerge. // // Notes: ResMan LOCKED // //---------------------------------------------------------------------------- void CMasterMerge::LokGrabResources( CDeletedIIDTrans & delIIDTrans ) { Win4Assert( mtMaster == _mt ); // ================================================= CStartMergeTrans xact( *this ); LokSetup( TRUE ); if (_indSnap.Count() == 0) { return; } Win4Assert( !_fSoftAbort ); { WORKID widDummy; _pPart->GetMMergeObjectIds(widDummy, widDummy, _widCurrentMaster); INDEXID iidNewKeyList = _resman._sKeyList->GetNextIid(); // // For down-level, the ObjectId for logs like the master merge log, // change log, etc are formed by concatenating the "it" value to // the partition-id. // CIndexId iidMMLog( itMMLog, _partid ); _widMasterLog = (INDEXID) iidMMLog; XWid xWidMasterLog( _widMasterLog, _resman._storage ); #ifdef KEYLIST_ENABLED _widKeyList = (INDEXID) iidNewKeyList; XWid xWidKeyList( _widKeyList, _resman._storage ); #endif // KEYLIST_ENABLED _resman._storage.InitRcovObj(_widMasterLog, FALSE); // // Ideally, this should be the order of steps: // 1. Create the KeyList and the New Master Index // 2. Store the master merge state. // // However, OFS imposes an ordering on the opening of objects. // We cannot open the IndexTable object for write access when // we have other ci objects (like an index or keylist) opened // for write access in the same thread. // // Because of this restriction, we have to update the index table // first which means we must be able to tolerate failures after // we have committed the start of a master merge in the index // table. // { #ifdef KEYLIST_ENABLED // // We don't want the creation of key list to fail after // we have created the index and hash streams and before // we have created the directory stream. So, first try to // create the entire key list object and if it succeeds we // don't have to deal with a partially constructed key list. // CWKeyList * pNewKeyList = new CWKeyList ( _resman._storage, _widKeyList, iidNewKeyList, _resman._sKeyList->Size(), _resman._sKeyList.GetPointer() ); //approx of size // // We must delete is here. O/W inversion of priority levels // will cause a deadlock in OFS. // delete pNewKeyList; #endif // KEYLIST_ENABLED // // Pre-Create the master index with all the streams to deal // with the possibility of failing after committing the start // of a master merge but before we succeeded in creating all // the streams. On a restart, the master merge index EXPECTS // to find all the necessary streams (index/dir). // // Create the new index with a size of 64k. This // reduces the free disk space required for a master merge to // approximately 64k. We'll be decommiting (shrink from front) // the old master index as the merge progress. // // Note that the initial size is in 4k CI pages. // unsigned c4kPages = 65536 / 4096; // 16 ciDebugOut(( DEB_ITRACE, "creating new master index of size 0x%x bytes\n", c4kPages * 4096 )); CPersIndex * pIndex = new CPersIndex( _resman._storage, _widNewIndex, _iidNew, c4kPages, CDiskIndex::eMaster ); delete pIndex; } ciFAILTEST(STATUS_NO_MEMORY); // // Store the master merge state. // LokStoreRestartResources( delIIDTrans ); // // Acquire the master log & keylist wids so that they are NOT deleted // from disk if an exception occurs. Once LokStoreRestartResources() // completes, any future exceptions should cause the merge to restart // from a 'paused' condition. // xWidMasterLog.Acquire(); #ifdef KEYLIST_ENABLED xWidKeyList.Acquire(); #endif // KEYLIST_ENABLED _fSoftAbort = TRUE; ciFAILTEST(STATUS_NO_MEMORY); // // After this point we are in a master merge and any failure // must be tolerated. Create a new master index. If there is a // failure while creating the in-memory structure, it can be created // later when merge is re-attempted. // LokCreateOrFindNewMaster(); ciFAILTEST(STATUS_NO_MEMORY); #ifdef KEYLIST_ENABLED // // Open the pre-created keylist. // CKeyBuf * pKeyLstSplitKey = new CKeyBuf(); SKeyBuf sKeyLstSplitKey(pKeyLstSplitKey); pKeyLstSplitKey->FillMin(); _pNewKeyList = new CWKeyList ( _resman._storage, _widKeyList, iidNewKeyList, _resman._sKeyList.GetPointer(), *pKeyLstSplitKey, _resman._sKeyList->MaxWorkId() ); #else _pNewKeyList = new CWKeyList( 0, iidNewKeyList); #endif // KEYLIST_ENABLED } ciFAILTEST(STATUS_NO_MEMORY); xact.Commit(); } void CMasterMerge::Do(CCiFrmPerfCounter & mergeProgress) { CMasterMergeIndex * pIndexNew = (CMasterMergeIndex *)_pIndexNew; pIndexNew->Merge ( _pNewKeyList, _indSnap.GetFresh(), *_pPart, mergeProgress, _resman.GetRegParams(), TRUE // Compute Relevant Words ); } CMasterMerge::~CMasterMerge() { Win4Assert( 0 == _pNewKeyList ); } //+--------------------------------------------------------------------------- // // Function: LokRollBack // // Synopsis: Rollsback a "paused" or "aborted" master merge. A master // merge is aborted only if it fails before it has been // recorded in the index table. Once the start has been // committted, it can only be "paused" and must be restarted // later. If _fSoftAbort is TRUE, then we have already committed // the start of master merge and should do a soft abort only. // // Arguments: [swapped] -- // // History: 8-30-94 srikants Created // // Notes: // //---------------------------------------------------------------------------- void CMasterMerge::LokRollBack( unsigned swapped ) { if ( _pNewKeyList ) { #if KEYLIST_ENABLED if ( !_fSoftAbort ) _pNewKeyList->Remove(); #endif // KEYLIST_ENABLED delete _pNewKeyList; _pNewKeyList = 0; } if ( _fSoftAbort ) { _pIndexNew = 0; return; } // // It is not a soft abort. Must completely abort the master merge. // CMerge::LokRollBack( swapped ); } //+--------------------------------------------------------------------------- // // Function: LokCreateOrFindNewMaster // // Synopsis: This routine either "creates" a new CMasterMergeIndex if // one doesn't already exist or use an existing one if it // already exists in the partition. // // History: 8-30-94 srikants Moved and adapted from CMerge. // // Notes: // //---------------------------------------------------------------------------- CMasterMergeIndex * CMasterMerge::LokCreateOrFindNewMaster() { Win4Assert( 0 == _pIndexNew ); CPersIndex * pCurrentMasterIndex = _pPart->GetCurrentMasterIndex(); _iidNew = _pPart->GetNewMasterIid(); CMasterMergeIndex * pIndexNew = 0; if ( 0 != pCurrentMasterIndex && pCurrentMasterIndex->GetId() == _iidNew ) { // // The New Master Index already exists and has been given // to the partition. We should use that. // Win4Assert( _pPart->GetOldMasterIndex() != pCurrentMasterIndex ); _pIndexNew = pCurrentMasterIndex; pIndexNew = (CMasterMergeIndex *)_pIndexNew; } else { CIndexSnapshot * pMergeIndSnap = _pPart->GetMMergeIndSnap(); Win4Assert( 0 != pMergeIndSnap ); // // The indexid of the new master index is different from the // current master index (if one exists). So, we are restarting // a stopped master merge which failed before the new master // index was created. // pIndexNew = new CMasterMergeIndex( _resman._storage, _widNewIndex, _iidNew, pMergeIndSnap->MaxWorkId(), pCurrentMasterIndex, _widMasterLog ); _pIndexNew = pIndexNew; pIndexNew->SetMMergeIndSnap( _pPart->AcquireMMergeIndSnap() ); _pPart->AddIndex ( _pIndexNew ); if ( 0 != pCurrentMasterIndex ) { _pPart->LokRemoveIndex( pCurrentMasterIndex->GetId() ); _pPart->SetOldMasterIndex( pCurrentMasterIndex ); } Win4Assert( _pIndexNew->IsMaster() ); Win4Assert( _pPart->GetCurrentMasterIndex() == _pIndexNew ); } return pIndexNew; } //LokCreateOrFindNewMaster //+--------------------------------------------------------------------------- // // Function: LokTakeIndexes // // Synopsis: Takes ownership of the merge indexes from the // CMasterMergeIndex. This is done at the end to complete the // commitment of the master merge and the indexsnapshot in the // merge object should have all the indexes that participated // in the merge. // // Arguments: [pMaster] -- Pointer to the CMasterMergeIndex. // // History: 9-29-94 srikants Created // // Notes: // //---------------------------------------------------------------------------- void CMasterMerge::LokTakeIndexes( CMasterMergeIndex * pMaster ) { _indSnap.LokTakeIndexes( pMaster->LokGetIndSnap() ); }