/*++ Copyright (c) 1996-1997 Microsoft Corporation Module Name: resfail.c Abstract: Cluster resource state management routines. Author: Mike Massa (mikemas) 14-Jan-1996 Revision History: --*/ #include "fmp.h" #define LOG_MODULE RESFAIL // globals // // Local Functions // DWORD FmpHandleResStateChangeProc( IN LPVOID pContext ); VOID FmpHandleResourceFailure( IN PFM_RESOURCE pResource ) /*++ Routine Description: Handles resource failure notifications from the resource monitor. Arguments: Resource - The resource which has failed. Return Value: None. Note: This routine is only called if the resource was online at the time of the failure. --*/ { DWORD dwStatus; BOOL bRestartGroup = TRUE; DWORD tickCount; DWORD withinFailurePeriod; CsLogEvent1(LOG_CRITICAL, FM_RESOURCE_FAILURE, OmObjectName(pResource) ); ClRtlLogPrint(LOG_NOISE, "[FM] FmpHandleResourceFailure: taking resource %1!ws! and dependents offline\n", OmObjectId(pResource)); if ( pResource->State == ClusterResourceOnline ) { ClRtlLogPrint(LOG_NOISE, "[FM] Resource %1!ws! failed, but still online!\n", OmObjectId(pResource)); } dwStatus = FmpTerminateResource(pResource); if (dwStatus != NO_ERROR) { ClRtlLogPrint(LOG_NOISE, "[FM] FmpHandleResourceFailure: offline of resource %1!ws! failed\n", OmObjectId(pResource)); } // // If system shutdown has begun, then don't bother trying to restart anything. // We can see spurious failures during shutdown as the network goes away, but // we do not want to be restarting resources while FmShutdownGroups is trying // to take them offline! // if (FmpShutdown) { return; } // SS: We handle the failure of the quorum resource specially // since other resources rely on it and may be blocked waiting // for the quorum resource to come online. ++ pResource->NumberOfFailures; switch ( pResource->RestartAction ) { case RestartNot: // Don't do anything. // However, if this is a quorum resource cause it to halt if (pResource->QuorumResource) { //cleanup quorum resource and cause the node to halt if (pResource->RestartAction == RestartNot) { FmpCleanupQuorumResource(pResource); CsInconsistencyHalt(ERROR_QUORUM_RESOURCE_ONLINE_FAILED); } } break; case RestartLocal: // fall through is correct for this case bRestartGroup = FALSE; case RestartGroup: // // If the number of failures is too high, then don't restart locally. // If this was a local restart then don't notify FM so that Group // doesn't move because of this guy; otherwise notify the FM that the // group has failed. // // // Get our current time, in milliseconds. // tickCount = GetTickCount(); // // Compute a boolean that tells if we are withing the allotted // failure period. // withinFailurePeriod = ( ((tickCount - pResource->FailureTime) <= pResource->RestartPeriod) ? TRUE : FALSE); // // If it's been a long time since our last failure, then // get the current time of this failure, and reset the count // of failures. // if ( !withinFailurePeriod ) { pResource->FailureTime = tickCount; pResource->NumberOfFailures = 1; } if ( pResource->NumberOfFailures <= pResource->RestartThreshold ) { FmpRestartResourceTree( pResource ); } else if ( bRestartGroup ) { ClusterEvent( CLUSTER_EVENT_GROUP_FAILED, pResource->Group ); } else { ClRtlLogPrint(LOG_NOISE, "[FM] RestartLocal: resource %1!ws! has exceeded its restart limit!\n", OmObjectId(pResource)); if (pResource->QuorumResource) { FmpCleanupQuorumResource(pResource); CsInconsistencyHalt(ERROR_QUORUM_RESOURCE_ONLINE_FAILED); } // Start a timer for which will attempt to restart the resource later FmpDelayedStartRes(pResource); } break; default: ClRtlLogPrint(LOG_NOISE,"[FM] FmpHandleResourceFailure: unknown restart action! Value = %1!u!\n", pResource->RestartAction); } return; } // FmpHandleResourceFailure VOID FmpHandleResourceTransition( IN PFM_RESOURCE Resource, IN CLUSTER_RESOURCE_STATE NewState ) /*++ Routine Description: Takes appropriate action based on resource state transitions indicated by the Resource Monitor. Arguments: Resource - The resource which has transitioned. NewState - The new state of Resource. Return Value: None. --*/ { DWORD status; DWORD dwOldBlockingFlag; ChkFMState: ACQUIRE_SHARED_LOCK(gQuoChangeLock); if (!FmpFMGroupsInited) { DWORD dwRetryCount = 50; //FmFormNewClusterPhaseProcessing is in progress if (FmpFMFormPhaseProcessing) { ClRtlLogPrint(LOG_CRITICAL, "[FM] FmpHandleResourceTransition: resource notification from quorum resource " "during phase processing. Sleep and retry\n"); RELEASE_LOCK(gQuoChangeLock); Sleep(500); if (dwRetryCount--) goto ChkFMState; else { ClRtlLogPrint(LOG_CRITICAL, "[FM] FmpHandleResourceTransition: waited for too long\n"); //terminate the process CL_ASSERT(FALSE); CsInconsistencyHalt(ERROR_CLUSTER_NODE_DOWN); } } //this can only come from the quorum resource CL_ASSERT(Resource->QuorumResource); } // if this is from the quorum resource, we need to do some special handling // protect the check for quorum resource by acquiring the shared lock if (Resource->QuorumResource) { // // Chittur Subbaraman (chitturs) - 6/25/99 // // Handle the sync notifications for the quorum resource. This is // done here instead of in FmpRmDoInterlockedDecrement since we // need to hold the gQuoChangeLock for this to synchronize with // other threads such as the FmCheckQuorumState called by the DM // node down handler. Note that FmpRmDoInterLockedDecrement needs // to be done with NO LOCKS held since it easily runs into deadlock // situations in which the quorum resource offline is waiting to // have the blocking resources count go to 0 and FmpRmDoInterLockedDecrement // which alone can make this count to 0 could be stuck waiting for // the lock. // DWORD dwBlockingFlag = InterlockedExchange( &Resource->BlockingQuorum, 0 ); CL_ASSERT( dwBlockingFlag == FALSE ); FmpCallResourceNotifyCb( Resource, NewState ); ACQUIRE_EXCLUSIVE_LOCK(gQuoLock); } else { FmpAcquireLocalResourceLock(Resource); } ClRtlLogPrint( NewState == ClusterResourceFailed ? LOG_UNUSUAL : LOG_NOISE, "[FM] HandleResourceTransition: Resource Name = %1!ws! old state=%2!u! new state=%3!u!\r\n", OmObjectId(Resource), Resource->State, NewState ); if ( Resource->State == NewState ) { ClRtlLogPrint(LOG_NOISE, "[FM] HandleResourceTransition: Resource %1!ws! already in state=%2!u!\r\n", OmObjectId(Resource), NewState ); goto FnExit; } switch (Resource->State) { case ClusterResourceOnline: // if there is a resource failure, then let the worker thread handle it // if there is a state change call the resource state change handler if (Resource->State != NewState) FmpPropagateResourceState( Resource, NewState ); if (NewState == ClusterResourceFailed) { if (Resource->QuorumResource) { RELEASE_LOCK(gQuoLock); FmpProcessResourceEvents(Resource, ClusterResourceFailed, ClusterResourceOnline); ACQUIRE_EXCLUSIVE_LOCK(gQuoLock); } else { FmpProcessResourceEvents(Resource, ClusterResourceFailed, ClusterResourceOnline); } } else { CL_ASSERT( (NewState == ClusterResourceOnline) || (NewState == ClusterResourceOffline) ); } break; case ClusterResourceFailed: if (Resource->State != NewState) FmpPropagateResourceState( Resource, NewState ); break; case ClusterResourceOfflinePending: //SS: a resource cannot go from one pending state to another CL_ASSERT( NewState < ClusterResourcePending ) // fall through case ClusterResourceOffline: // // Because this resource is now unstuck... there may be other // pending threads waiting to clear up. If not, they'll just get // stuck again, until the next notification. // switch ( NewState ) { case ClusterResourceFailed: if ( Resource->State != NewState ) FmpPropagateResourceState( Resource, NewState ); // if it is the quorum resource handle the locking appropriately if (Resource->QuorumResource) { // // Chittur Subbaraman (chitturs) - 9/20/99 // // Release and reacquire the gQuoLock to maintain // locking order between group lock and gQuoLock. // RELEASE_LOCK(gQuoLock); FmpProcessResourceEvents(Resource, ClusterResourceFailed, ClusterResourceOffline); ACQUIRE_EXCLUSIVE_LOCK(gQuoLock); } else { FmpProcessResourceEvents(Resource, ClusterResourceFailed, ClusterResourceOffline); } break; case ClusterResourceOffline: if ( Resource->Group->OwnerNode == NmLocalNode ) { if ( Resource->State != NewState ) { FmpPropagateResourceState( Resource, NewState ); } // if it is the quorum resource handle the locking appropriately if (Resource->QuorumResource) { // // Chittur Subbaraman (chitturs) - 9/20/99 // // Release and reacquire the gQuoLock to maintain // locking order between group lock and gQuoLock. // RELEASE_LOCK(gQuoLock); FmpProcessResourceEvents(Resource, ClusterResourceOffline, ClusterResourceOfflinePending); ACQUIRE_EXCLUSIVE_LOCK(gQuoLock); } else { FmpProcessResourceEvents(Resource, ClusterResourceOffline, ClusterResourceOfflinePending); } } else { if ( Resource->State != NewState ) { FmpPropagateResourceState( Resource, NewState ); } } break; default: if ( Resource->State != NewState ) { FmpPropagateResourceState( Resource, NewState ); } break; } break; case ClusterResourceOnlinePending: //SS: a resource cannot go from one pending state to another CL_ASSERT( NewState < ClusterResourcePending ) // // Because this resource is now unstuck... there may be other // pending threads waiting to clear up. If not, they'll just get // stuck again, until the next notification. // switch ( NewState ) { case ClusterResourceFailed: // // Make sure we go through full failure recovery. // //SS: dont know why the state is being set to online //it could be online pending //Resource->State = ClusterResourceOnline; ClRtlLogPrint(LOG_NOISE, "[FM] HandleResourceTransition: Resource failed, post a work item\r\n"); if (Resource->State != NewState) FmpPropagateResourceState( Resource, NewState ); // since this is the quorum Resource handle locking appropriately if (Resource->QuorumResource) { // // Chittur Subbaraman (chitturs) - 9/20/99 // // Release and reacquire the gQuoLock to maintain // locking order between group lock and gQuoLock. // RELEASE_LOCK(gQuoLock); FmpProcessResourceEvents(Resource, ClusterResourceFailed, ClusterResourceOnlinePending); ACQUIRE_EXCLUSIVE_LOCK(gQuoLock); } else { FmpProcessResourceEvents(Resource, ClusterResourceFailed, ClusterResourceOnlinePending); } break; case ClusterResourceOnline: if (Resource->Group->OwnerNode == NmLocalNode) { //Call FmpPropagateResourceState without holding the group //lock for the quorum resource FmpPropagateResourceState( Resource, NewState ); // since this is the quorum Resource fork another thread if (Resource->QuorumResource) { // // Chittur Subbaraman (chitturs) - 9/20/99 // // Release and reacquire the gQuoLock to maintain // locking order between group lock and gQuoLock. // RELEASE_LOCK(gQuoLock); FmpProcessResourceEvents(Resource, ClusterResourceOnline, ClusterResourceOnlinePending); ACQUIRE_EXCLUSIVE_LOCK(gQuoLock); } else { FmpProcessResourceEvents(Resource, ClusterResourceOnline, ClusterResourceOnlinePending); } } else { FmpPropagateResourceState( Resource, NewState ); } break; default: if (Resource->State != NewState) FmpPropagateResourceState( Resource, NewState ); break; } Resource->Flags &= ~RESOURCE_WAITING; break; case ClusterResourceInitializing: default: if (Resource->State != NewState) FmpPropagateResourceState( Resource, NewState ); CL_ASSERT(Resource->State == NewState); } FnExit: if (Resource->QuorumResource) { RELEASE_LOCK(gQuoLock); } else { FmpReleaseLocalResourceLock(Resource); } RELEASE_LOCK(gQuoChangeLock); return; } /**** @func DWORD | FmpCreateResNotificationHandler| This creates a new thread to handle state change notifications for the given resource. @parm IN PFM_RESOURCE | pResource | Pointer to the resource. @parm IN CLUSTER_RESOURCE_STATE | OldState | The old state of the resource from which it transitioned. @parm IN CLUSTER_RESOURCE_STATE | NewState | The new state of the resource. @comm This routine creates a thread to perform all the pending work when the resource changes state that cannot be performed within FmpHandleResourceTransition to avoid deadlocks and that cannot be deffered to the FmpWorkerThread because of serialization issues. In particular, it is used to handle state transition work for the quorum resource since other resources depend on the quorum resource and cannot come online till the state of the quorum becomes online. For instance, the quorum resource may be coming offline as a part of move while another resource if in FmpWorkerThread() calling FmpOffline/OnlineWaitingTree(). For the quorum resource to come online again (that happens by signalling the move pending thread) so that FmpWorkerThread can make progress its events will have to be handled separately. @rdesc Returns a result code. ERROR_SUCCESS on success. @xref ****/ DWORD FmpCreateResStateChangeHandler( IN PFM_RESOURCE pResource, IN CLUSTER_RESOURCE_STATE NewState, IN CLUSTER_RESOURCE_STATE OldState) { HANDLE hThread = NULL; DWORD dwThreadId; PRESOURCE_STATE_CHANGE pResStateContext = NULL; DWORD dwStatus = ERROR_SUCCESS; //reference the resource //the thread will dereference it, if the thread is successfully //created ClRtlLogPrint(LOG_NOISE, "[FM] FmpCreateResStateChangeHandler: Entry\r\n"); OmReferenceObject(pResource); pResStateContext = LocalAlloc(LMEM_FIXED, sizeof(RESOURCE_STATE_CHANGE)); if (!pResStateContext) { dwStatus = GetLastError(); CL_UNEXPECTED_ERROR(dwStatus); goto FnExit; } pResStateContext->pResource = pResource; pResStateContext->OldState = OldState; pResStateContext->NewState = NewState; hThread = CreateThread( NULL, 0, FmpHandleResStateChangeProc, pResStateContext, 0, &dwThreadId ); if ( hThread == NULL ) { dwStatus = GetLastError(); CL_UNEXPECTED_ERROR(dwStatus); // if the function failed to create the thread, cleanup the // state that the thread would have cleaned //deref the object if the thread is not created successfully OmDereferenceObject(pResource); LocalFree(pResStateContext); goto FnExit; } FnExit: //do general cleanup if (hThread) CloseHandle(hThread); ClRtlLogPrint(LOG_NOISE, "[FM] FmpCreateResStateChangeHandler: Exit, status %1!u!\r\n", dwStatus); return(dwStatus); } /**** @func DWORD | FmpHandleResStateChangeProc| This thread procedure handles all the post processing for the resource transitions for the quorum resource. @parm IN LPVOID | pContext | A pointer to PRESOURCE_STATE_CHANGE structure. @comm This thread handles a resource change notification postprocessing. Significantly for quorum resource so that quorum resource state change notifications are not handled by the single FmpWorkThread() [that causes deadlock - if the quorum notification resource is queued behind a notification whose handling requires tha quorum resource be online].. @rdesc Returns a result code. ERROR_SUCCESS on success. @xref pResource, pResStateChange->NewState ); OmDereferenceObject( pResStateChange->pResource ); LocalFree( pResStateChange ); ClRtlLogPrint(LOG_NOISE, "[FM] FmpHandleResStateChangeProc: Exit...\r\n"); return( ERROR_SUCCESS ); } DWORD FmpDelayedStartRes( IN PFM_RESOURCE pResource ) /*++ Routine Description: Starts a timer for the resource. FmpDelayedRestartCb function will be invoked at the expiry of timer.. Arguments: pResource - The resource which has transitioned. Return Value: ERROR_SUCCESS if successful, WIN32 errorcode otherwise. Note that no delayed restart attempts are made if the resource is a quorum resource. --*/ { DWORD dwStatus = ERROR_SUCCESS; ClRtlLogPrint(LOG_NOISE, "[FM] FmpDelayedRestartRes:Entry for resource %1!ws!\n", OmObjectId(pResource)); if( (pResource->RetryPeriodOnFailure != CLUSTER_RESOURCE_DEFAULT_RETRY_PERIOD_ON_FAILURE ) && !(pResource->QuorumResource) ) { // Check if there is already a timer running for this resource if(pResource->hTimer == NULL) { pResource->hTimer = CreateWaitableTimer(NULL, FALSE, NULL); if (!(pResource->hTimer)) { // not a fatal error but log it ClRtlLogPrint(LOG_ERROR, "[FM] FmpDelayedRestartRes: failed to create the watchdog timer for resource %1!ws!\n", OmObjectId(pResource)); } else{ ClRtlLogPrint(LOG_NOISE, "[FM] FmpDelayedRestartRes: Adding watchdog timer for resource %1!ws!, period=%2!u!\n", OmObjectId(pResource), pResource->RetryPeriodOnFailure); // make sure resource struct won't go away if resource is deleted before the timer fires OmReferenceObject(pResource); //register the timer with the periodic activity timer thread dwStatus = AddTimerActivity(pResource->hTimer, pResource->RetryPeriodOnFailure, 0, FmpDelayedRestartCb, pResource); if (dwStatus != ERROR_SUCCESS) { ClRtlLogPrint(LOG_ERROR, "[FM] FmpDelayedRestartRes: AddTimerActivity failed with error %1!u!\n", dwStatus); CloseHandle(pResource->hTimer); pResource->hTimer = NULL; } } } } return dwStatus; } VOID FmpDelayedRestartCb( IN HANDLE hTimer, IN PVOID pContext) /*++ Routine Description This is invoked by timer activity thread to attempt a restart on a failed resource. Arguments pContext - a pointer to PFM_RESOURCE Return Value ERROR_SUCCESS on success, a WIN32 error code otherwise. --*/ { PFM_RESOURCE pResource; pResource=(PFM_RESOURCE)pContext; ClRtlLogPrint(LOG_NOISE, "[FM] FmpDelayedRestartCb: Entry for resource %1!ws! \n", OmObjectId(pResource)); OmReferenceObject(pResource); FmpPostWorkItem(FM_EVENT_RES_RETRY_TIMER, pResource, 0); OmDereferenceObject(pResource); return; }