/*++ Copyright (c) 1989 Microsoft Corporation Copyright (c) 1990 Microsoft Corporation Module Name: tbflush.c Abstract: This module implements machine dependent functions to flush the translation buffers in an Intel x86 system. N.B. This module contains only MP versions of the TB flush routines. The UP versions are macros in ke.h KeFlushEntireTb remains a routine for the UP system since it is exported from the kernel for backwards compatibility. Author: David N. Cutler (davec) 13-May-1989 Environment: Kernel mode only. Revision History: Shie-Lin Tzong (shielint) 30-Aug-1990 Implement MP version of KeFlushSingleTb and KeFlushEntireTb. --*/ #include "ki.h" VOID KiFlushTargetEntireTb ( IN PKIPI_CONTEXT SignalDone, IN PVOID Invalid, IN PVOID Parameter2, IN PVOID Parameter3 ); VOID KiFlushTargetMultipleTb ( IN PKIPI_CONTEXT SignalDone, IN PVOID Parameter1, IN PVOID Parameter2, IN PVOID Parameter3 ); VOID KiFlushTargetSingleTb ( IN PKIPI_CONTEXT SignalDone, IN PVOID Parameter1, IN PVOID Parameter2, IN PVOID Parameter3 ); HARDWARE_PTE KiFlushSingleTbSynchronous ( IN PVOID Virtual, IN BOOLEAN Invalid, IN BOOLEAN AllProcessors, IN PHARDWARE_PTE PtePointer, IN HARDWARE_PTE PteValue ); VOID KiFlushTargetSingleTbSynchronous ( IN PKIPI_CONTEXT SignalDone, IN PVOID Parameter1, IN PVOID Parameter2, IN PVOID Parameter3 ); VOID Ki386UseSynchronousTbFlush ( IN volatile PLONG Number ); #ifdef ALLOC_PRAGMA #pragma alloc_text(INIT,Ki386UseSynchronousTbFlush) #endif #if !defined(NT_UP) VOID KeFlushEntireTb ( IN BOOLEAN Invalid, IN BOOLEAN AllProcessors ) /*++ Routine Description: This function flushes the entire translation buffer (TB) on all processors that are currently running threads which are child of the current process or flushes the entire translation buffer on all processors in the host configuration. Arguments: Invalid - Supplies a boolean value that specifies the reason for flushing the translation buffer. AllProcessors - Supplies a boolean value that determines which translation buffers are to be flushed. Return Value: None. --*/ { KAFFINITY EntireSet; KIRQL OldIrql; PKPRCB Prcb; PKPROCESS Process; KAFFINITY TargetProcessors; // // Compute the target set of processors, disable context switching, // and send the flush entire parameters to the target processors, // if any, for execution. // if (AllProcessors != FALSE) { OldIrql = KeRaiseIrqlToSynchLevel(); Prcb = KeGetCurrentPrcb(); TargetProcessors = KeActiveProcessors; } else { KiLockContextSwap(&OldIrql); Prcb = KeGetCurrentPrcb(); Process = Prcb->CurrentThread->ApcState.Process; TargetProcessors = Process->ActiveProcessors; } EntireSet = KeActiveProcessors & ~Prcb->SetMember; TargetProcessors &= ~Prcb->SetMember; // // If the target set of processors is equal to the full set of processors, // then set the TB flush time stamp busy. // if (TargetProcessors == EntireSet) { KiSetTbFlushTimeStampBusy(); } // // Send packet to target processors. // if (TargetProcessors != 0) { KiIpiSendPacket(TargetProcessors, KiFlushTargetEntireTb, NULL, NULL, NULL); IPI_INSTRUMENT_COUNT (Prcb->Number, FlushEntireTb); } // // Flush TB on current processor. // KeFlushCurrentTb(); // // Wait until all target processors have finished and complete packet. // if (TargetProcessors != 0) { KiIpiStallOnPacketTargets(TargetProcessors); } // // If the target set of processors is equal to the full set of processors, // then clear the TB flush time stamp busy. // if (TargetProcessors == EntireSet) { KiClearTbFlushTimeStampBusy(); } // // Lower IRQL and unlock as appropriate. // if (AllProcessors != FALSE) { KeLowerIrql(OldIrql); } else { KiUnlockContextSwap(OldIrql); } return; } VOID KiFlushTargetEntireTb ( IN PKIPI_CONTEXT SignalDone, IN PVOID Parameter1, IN PVOID Parameter2, IN PVOID Parameter3 ) /*++ Routine Description: This is the target function for flushing the entire TB. Arguments: SignalDone - Supplies a pointer to a variable that is cleared when the requested operation has been performed. Parameter1 - Parameter3 - Not used. Return Value: None. --*/ { // // Flush the entire TB on the current processor. // KiIpiSignalPacketDone(SignalDone); KeFlushCurrentTb(); return; } VOID KeFlushMultipleTb ( IN ULONG Number, IN PVOID *Virtual, IN BOOLEAN Invalid, IN BOOLEAN AllProcessors, IN PHARDWARE_PTE *PtePointer OPTIONAL, IN HARDWARE_PTE PteValue ) /*++ Routine Description: This function flushes multiple entries from the translation buffer on all processors that are currently running threads which are children of the current process or flushes a multiple entries from the translation buffer on all processors in the host configuration. Arguments: Number - Supplies the number of TB entries to flush. Virtual - Supplies a pointer to an array of virtual addresses that are within the pages whose translation buffer entries are to be flushed. Invalid - Supplies a boolean value that specifies the reason for flushing the translation buffer. AllProcessors - Supplies a boolean value that determines which translation buffers are to be flushed. PtePointer - Supplies an optional pointer to an array of pointers to page table entries that receive the specified page table entry value. PteValue - Supplies the the new page table entry value. Return Value: The previous contents of the specified page table entry is returned as the function value. --*/ { ULONG Index; KIRQL OldIrql; PKPRCB Prcb; PKPROCESS Process; KAFFINITY TargetProcessors; // // Compute target set of processors. // if (AllProcessors != FALSE) { OldIrql = KeRaiseIrqlToSynchLevel(); Prcb = KeGetCurrentPrcb(); TargetProcessors = KeActiveProcessors; } else { KiLockContextSwap(&OldIrql); Prcb = KeGetCurrentPrcb(); Process = Prcb->CurrentThread->ApcState.Process; TargetProcessors = Process->ActiveProcessors; } TargetProcessors &= ~Prcb->SetMember; // // If a page table entry address array is specified, then set the // specified page table entries to the specific value. // if (ARGUMENT_PRESENT(PtePointer)) { for (Index = 0; Index < Number; Index += 1) { KI_FILL_PTE(PtePointer[Index], PteValue); } } // // If any target processors are specified, then send a flush multiple // packet to the target set of processors. // if (TargetProcessors != 0) { KiIpiSendPacket(TargetProcessors, KiFlushTargetMultipleTb, (PVOID)Invalid, (PVOID)Number, (PVOID)Virtual); IPI_INSTRUMENT_COUNT (Prcb->Number, FlushMultipleTb); } // // Flush the specified entries from the TB on the current processor. // for (Index = 0; Index < Number; Index += 1) { KiFlushSingleTb(Invalid, Virtual[Index]); } // // Wait until all target processors have finished and complete packet. // if (TargetProcessors != 0) { KiIpiStallOnPacketTargets(TargetProcessors); } // // Release the context swap lock. // if (AllProcessors != FALSE) { KeLowerIrql(OldIrql); } else { KiUnlockContextSwap(OldIrql); } return; } VOID KiFlushTargetMultipleTb ( IN PKIPI_CONTEXT SignalDone, IN PVOID Invalid, IN PVOID Number, IN PVOID Virtual ) /*++ Routine Description: This is the target function for flushing multiple TB entries. Arguments: SignalDone - Supplies a pointer to a variable that is cleared when the requested operation has been performed. Invalid - Supplies a bollean value that determines whether the virtual address is invalid. Number - Supplies the number of TB entries to flush. Virtual - Supplies a pointer to an array of virtual addresses that are within the pages whose translation buffer entries are to be flushed. Return Value: None. --*/ { ULONG Index; PVOID VirtualAddress[FLUSH_MULTIPLE_MAXIMUM]; // // Capture the virtual addresses that are to be flushed from the TB // on the current processor and signal pack done. // for (Index = 0; Index < (ULONG) Number; Index += 1) { VirtualAddress[Index] = ((PVOID *)(Virtual))[Index]; } KiIpiSignalPacketDone(SignalDone); // // Flush the specified virtual address for the TB on the current // processor. // for (Index = 0; Index < (ULONG) Number; Index += 1) { KiFlushSingleTb((BOOLEAN)Invalid, VirtualAddress [Index]); } } HARDWARE_PTE KeFlushSingleTb ( IN PVOID Virtual, IN BOOLEAN Invalid, IN BOOLEAN AllProcessors, IN PHARDWARE_PTE PtePointer, IN HARDWARE_PTE PteValue ) /*++ Routine Description: This function flushes a single entry from translation buffer (TB) on all processors that are currently running threads which are child of the current process or flushes the entire translation buffer on all processors in the host configuration. Arguments: Virtual - Supplies a virtual address that is within the page whose translation buffer entry is to be flushed. Invalid - Supplies a boolean value that specifies the reason for flushing the translation buffer. AllProcessors - Supplies a boolean value that determines which translation buffers are to be flushed. PtePointer - Address of Pte to update with new value. PteValue - New value to put in the Pte. Will simply be assigned to *PtePointer, in a fashion correct for the hardware. Return Value: Returns the contents of the PtePointer before the new value is stored. --*/ { KIRQL OldIrql; PKPRCB Prcb; PKPROCESS Process; HARDWARE_PTE OldPteValue; KAFFINITY TargetProcessors; // // Compute target set of processors. // if (AllProcessors != FALSE) { OldIrql = KeRaiseIrqlToSynchLevel(); Prcb = KeGetCurrentPrcb(); TargetProcessors = KeActiveProcessors; } else { KiLockContextSwap(&OldIrql); Prcb = KeGetCurrentPrcb(); Process = Prcb->CurrentThread->ApcState.Process; TargetProcessors = Process->ActiveProcessors; } TargetProcessors &= ~Prcb->SetMember; // // Capture the previous contents of the page table entry and set the // page table entry to the new value. // KI_SWAP_PTE(PtePointer, PteValue, OldPteValue); // // If any target processors are specified, then send a flush single // packet to the target set of processors. // if (TargetProcessors != 0) { KiIpiSendPacket(TargetProcessors, KiFlushTargetSingleTb, (PVOID)Invalid, (PVOID)Virtual, NULL); IPI_INSTRUMENT_COUNT(Prcb->Number, FlushSingleTb); } // // Flush the specified entry from the TB on the current processor. // KiFlushSingleTb(Invalid, Virtual); // // Wait until all target processors have finished and complete packet. // if (TargetProcessors != 0) { KiIpiStallOnPacketTargets(TargetProcessors); } // // Release the context swap lock. // if (AllProcessors != FALSE) { KeLowerIrql(OldIrql); } else { KiUnlockContextSwap(OldIrql); } return(OldPteValue); } VOID KiFlushTargetSingleTb ( IN PKIPI_CONTEXT SignalDone, IN PVOID Invalid, IN PVOID VirtualAddress, IN PVOID Parameter3 ) /*++ Routine Description: This is the target function for flushing a single TB entry. Arguments: SignalDone Supplies a pointer to a variable that is cleared when the requested operation has been performed. Invalid - Supplies a bollean value that determines whether the virtual address is invalid. Virtual - Supplies a virtual address that is within the page whose translation buffer entry is to be flushed. Parameter3 - Not used. Return Value: None. --*/ { // // Flush a single entry from the TB on the current processor. // KiIpiSignalPacketDone(SignalDone); KiFlushSingleTb((BOOLEAN)Invalid, (PVOID)VirtualAddress); } HARDWARE_PTE KiFlushSingleTbSynchronous ( IN PVOID Virtual, IN BOOLEAN Invalid, IN BOOLEAN AllProcessors, IN PHARDWARE_PTE PtePointer, IN HARDWARE_PTE PteValue ) /*++ Routine Description: This function is a slow synchronous version of KeFlushSingleTb. We need this function as many P6's don't actually know how to deal with PTEs in an MP safe manner. Arguments: See KeFlushSingleTb Return Value: See KeFlushSingleTb --*/ { KIRQL OldIrql; PKPRCB Prcb; PKPROCESS Process; HARDWARE_PTE OldPteValue; KAFFINITY TargetProcessors; // // Synchronize will all other single flush calls (and other // IPIs which use reverse stalls) // KiLockContextSwap(&OldIrql); // // Compute target set of processors. // Prcb = KeGetCurrentPrcb(); if (AllProcessors != FALSE) { TargetProcessors = KeActiveProcessors; } else { Process = Prcb->CurrentThread->ApcState.Process; TargetProcessors = Process->ActiveProcessors; } TargetProcessors &= ~Prcb->SetMember; // // If any target processors are specified, then send a flush single // packet to the target set of processors. // if (TargetProcessors != 0) { KiIpiSendSynchronousPacket(Prcb, TargetProcessors, KiFlushTargetSingleTbSynchronous, (PVOID)Invalid, (PVOID)Virtual, (PVOID)&Prcb->ReverseStall ); IPI_INSTRUMENT_COUNT(Prcb->Number, FlushSingleTb); // // Wait for the target processors to stall // KiIpiStallOnPacketTargets(TargetProcessors); // // Capture the previous contents of the page table entry and set the // page table entry to the new value. // KI_SWAP_PTE(PtePointer, PteValue, OldPteValue); // // Notify all prcessors it's time to go // Prcb->ReverseStall += 1; } else { // // Capture the previous contents of the page table entry and set the // page table entry to the new value. // KI_SWAP_PTE(PtePointer, PteValue, OldPteValue); } // // Flush the specified entry from the TB on the current processor. // KiFlushSingleTb(Invalid, Virtual); // // Done // KiUnlockContextSwap(OldIrql); return(OldPteValue); } VOID KiFlushTargetSingleTbSynchronous ( IN PKIPI_CONTEXT SignalDone, IN PVOID Invalid, IN PVOID VirtualAddress, IN PVOID Proceed ) /*++ Routine Description: This is the target function for flushing a single TB entry synchronously. Arguments: SignalDone Supplies a pointer to a variable that is cleared when the requested operation has been performed. Invalid - Supplies a bollean value that determines whether the virtual address is invalid. Virtual - Supplies a virtual address that is within the page whose translation buffer entry is to be flushed. Parameter3 - Not used. Return Value: None. --*/ { // // Flush a single entry from the TB on the current processor. // KiIpiSignalPacketDoneAndStall(SignalDone, Proceed); KiFlushSingleTb((BOOLEAN)Invalid, (PVOID)VirtualAddress); } VOID Ki386UseSynchronousTbFlush ( IN volatile PLONG Number ) { PKPRCB Prcb; volatile PUCHAR Patch; Prcb = KeGetCurrentPrcb(); Patch = (PUCHAR) KeFlushSingleTb; // // Signal we're here and wait for others // InterlockedDecrement (Number); while (*Number) ; // // If this is processor 0 apply the patch // if (Prcb->Number == 0) { *((PULONG) &Patch[1]) = ((ULONG) &KiFlushSingleTbSynchronous) - ((ULONG) Patch) - 5; Patch[0] = 0xe9; } // // Wait for processor 0 to complete installation of handler // while (Patch[0] != 0xe9) ; } #endif