791 lines
17 KiB
C
791 lines
17 KiB
C
/*++
|
||
|
||
Copyright (c) 1989 Microsoft Corporation
|
||
Copyright (c) 1990 Microsoft Corporation
|
||
|
||
Module Name:
|
||
|
||
tbflush.c
|
||
|
||
Abstract:
|
||
|
||
This module implements machine dependent functions to flush
|
||
the translation buffers in an Intel x86 system.
|
||
|
||
N.B. This module contains only MP versions of the TB flush routines.
|
||
The UP versions are macros in ke.h
|
||
KeFlushEntireTb remains a routine for the UP system since it is
|
||
exported from the kernel for backwards compatibility.
|
||
|
||
Author:
|
||
|
||
David N. Cutler (davec) 13-May-1989
|
||
|
||
Environment:
|
||
|
||
Kernel mode only.
|
||
|
||
Revision History:
|
||
|
||
Shie-Lin Tzong (shielint) 30-Aug-1990
|
||
Implement MP version of KeFlushSingleTb and KeFlushEntireTb.
|
||
|
||
--*/
|
||
|
||
#include "ki.h"
|
||
|
||
VOID
|
||
KiFlushTargetEntireTb (
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Invalid,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
);
|
||
|
||
VOID
|
||
KiFlushTargetMultipleTb (
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Parameter1,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
);
|
||
|
||
|
||
VOID
|
||
KiFlushTargetSingleTb (
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Parameter1,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
);
|
||
|
||
HARDWARE_PTE
|
||
KiFlushSingleTbSynchronous (
|
||
IN PVOID Virtual,
|
||
IN BOOLEAN Invalid,
|
||
IN BOOLEAN AllProcessors,
|
||
IN PHARDWARE_PTE PtePointer,
|
||
IN HARDWARE_PTE PteValue
|
||
);
|
||
|
||
VOID
|
||
KiFlushTargetSingleTbSynchronous (
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Parameter1,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
);
|
||
|
||
VOID
|
||
Ki386UseSynchronousTbFlush (
|
||
IN volatile PLONG Number
|
||
);
|
||
|
||
#ifdef ALLOC_PRAGMA
|
||
#pragma alloc_text(INIT,Ki386UseSynchronousTbFlush)
|
||
#endif
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
|
||
VOID
|
||
KeFlushEntireTb (
|
||
IN BOOLEAN Invalid,
|
||
IN BOOLEAN AllProcessors
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function flushes the entire translation buffer (TB) on all processors
|
||
that are currently running threads which are child of the current process
|
||
or flushes the entire translation buffer on all processors in the host
|
||
configuration.
|
||
|
||
Arguments:
|
||
|
||
Invalid - Supplies a boolean value that specifies the reason for flushing
|
||
the translation buffer.
|
||
|
||
AllProcessors - Supplies a boolean value that determines which translation
|
||
buffers are to be flushed.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
KAFFINITY EntireSet;
|
||
KIRQL OldIrql;
|
||
PKPRCB Prcb;
|
||
PKPROCESS Process;
|
||
KAFFINITY TargetProcessors;
|
||
|
||
//
|
||
// Compute the target set of processors, disable context switching,
|
||
// and send the flush entire parameters to the target processors,
|
||
// if any, for execution.
|
||
//
|
||
|
||
if (AllProcessors != FALSE) {
|
||
OldIrql = KeRaiseIrqlToSynchLevel();
|
||
Prcb = KeGetCurrentPrcb();
|
||
TargetProcessors = KeActiveProcessors;
|
||
|
||
} else {
|
||
KiLockContextSwap(&OldIrql);
|
||
Prcb = KeGetCurrentPrcb();
|
||
Process = Prcb->CurrentThread->ApcState.Process;
|
||
TargetProcessors = Process->ActiveProcessors;
|
||
}
|
||
|
||
EntireSet = KeActiveProcessors & ~Prcb->SetMember;
|
||
TargetProcessors &= ~Prcb->SetMember;
|
||
|
||
//
|
||
// If the target set of processors is equal to the full set of processors,
|
||
// then set the TB flush time stamp busy.
|
||
//
|
||
|
||
if (TargetProcessors == EntireSet) {
|
||
KiSetTbFlushTimeStampBusy();
|
||
}
|
||
|
||
//
|
||
// Send packet to target processors.
|
||
//
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiSendPacket(TargetProcessors,
|
||
KiFlushTargetEntireTb,
|
||
NULL,
|
||
NULL,
|
||
NULL);
|
||
|
||
IPI_INSTRUMENT_COUNT (Prcb->Number, FlushEntireTb);
|
||
}
|
||
|
||
//
|
||
// Flush TB on current processor.
|
||
//
|
||
|
||
KeFlushCurrentTb();
|
||
|
||
//
|
||
// Wait until all target processors have finished and complete packet.
|
||
//
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiStallOnPacketTargets(TargetProcessors);
|
||
}
|
||
|
||
//
|
||
// If the target set of processors is equal to the full set of processors,
|
||
// then clear the TB flush time stamp busy.
|
||
//
|
||
|
||
if (TargetProcessors == EntireSet) {
|
||
KiClearTbFlushTimeStampBusy();
|
||
}
|
||
|
||
//
|
||
// Lower IRQL and unlock as appropriate.
|
||
//
|
||
|
||
if (AllProcessors != FALSE) {
|
||
KeLowerIrql(OldIrql);
|
||
|
||
} else {
|
||
KiUnlockContextSwap(OldIrql);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
|
||
|
||
VOID
|
||
KiFlushTargetEntireTb (
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Parameter1,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This is the target function for flushing the entire TB.
|
||
|
||
Arguments:
|
||
|
||
SignalDone - Supplies a pointer to a variable that is cleared when the
|
||
requested operation has been performed.
|
||
|
||
Parameter1 - Parameter3 - Not used.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
//
|
||
// Flush the entire TB on the current processor.
|
||
//
|
||
|
||
KiIpiSignalPacketDone(SignalDone);
|
||
KeFlushCurrentTb();
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
KeFlushMultipleTb (
|
||
IN ULONG Number,
|
||
IN PVOID *Virtual,
|
||
IN BOOLEAN Invalid,
|
||
IN BOOLEAN AllProcessors,
|
||
IN PHARDWARE_PTE *PtePointer OPTIONAL,
|
||
IN HARDWARE_PTE PteValue
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function flushes multiple entries from the translation buffer
|
||
on all processors that are currently running threads which are
|
||
children of the current process or flushes a multiple entries from
|
||
the translation buffer on all processors in the host configuration.
|
||
|
||
Arguments:
|
||
|
||
Number - Supplies the number of TB entries to flush.
|
||
|
||
Virtual - Supplies a pointer to an array of virtual addresses that
|
||
are within the pages whose translation buffer entries are to be
|
||
flushed.
|
||
|
||
Invalid - Supplies a boolean value that specifies the reason for
|
||
flushing the translation buffer.
|
||
|
||
AllProcessors - Supplies a boolean value that determines which
|
||
translation buffers are to be flushed.
|
||
|
||
PtePointer - Supplies an optional pointer to an array of pointers to
|
||
page table entries that receive the specified page table entry
|
||
value.
|
||
|
||
PteValue - Supplies the the new page table entry value.
|
||
|
||
Return Value:
|
||
|
||
The previous contents of the specified page table entry is returned
|
||
as the function value.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
ULONG Index;
|
||
KIRQL OldIrql;
|
||
PKPRCB Prcb;
|
||
PKPROCESS Process;
|
||
KAFFINITY TargetProcessors;
|
||
|
||
//
|
||
// Compute target set of processors.
|
||
//
|
||
|
||
if (AllProcessors != FALSE) {
|
||
OldIrql = KeRaiseIrqlToSynchLevel();
|
||
Prcb = KeGetCurrentPrcb();
|
||
TargetProcessors = KeActiveProcessors;
|
||
|
||
} else {
|
||
KiLockContextSwap(&OldIrql);
|
||
Prcb = KeGetCurrentPrcb();
|
||
Process = Prcb->CurrentThread->ApcState.Process;
|
||
TargetProcessors = Process->ActiveProcessors;
|
||
}
|
||
|
||
TargetProcessors &= ~Prcb->SetMember;
|
||
|
||
//
|
||
// If a page table entry address array is specified, then set the
|
||
// specified page table entries to the specific value.
|
||
//
|
||
|
||
if (ARGUMENT_PRESENT(PtePointer)) {
|
||
for (Index = 0; Index < Number; Index += 1) {
|
||
KI_FILL_PTE(PtePointer[Index], PteValue);
|
||
}
|
||
}
|
||
|
||
//
|
||
// If any target processors are specified, then send a flush multiple
|
||
// packet to the target set of processors.
|
||
//
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiSendPacket(TargetProcessors,
|
||
KiFlushTargetMultipleTb,
|
||
(PVOID)Invalid,
|
||
(PVOID)Number,
|
||
(PVOID)Virtual);
|
||
|
||
IPI_INSTRUMENT_COUNT (Prcb->Number, FlushMultipleTb);
|
||
}
|
||
|
||
//
|
||
// Flush the specified entries from the TB on the current processor.
|
||
//
|
||
|
||
for (Index = 0; Index < Number; Index += 1) {
|
||
KiFlushSingleTb(Invalid, Virtual[Index]);
|
||
}
|
||
|
||
//
|
||
// Wait until all target processors have finished and complete packet.
|
||
//
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiStallOnPacketTargets(TargetProcessors);
|
||
}
|
||
|
||
//
|
||
// Release the context swap lock.
|
||
//
|
||
|
||
if (AllProcessors != FALSE) {
|
||
KeLowerIrql(OldIrql);
|
||
|
||
} else {
|
||
KiUnlockContextSwap(OldIrql);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
KiFlushTargetMultipleTb (
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Invalid,
|
||
IN PVOID Number,
|
||
IN PVOID Virtual
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This is the target function for flushing multiple TB entries.
|
||
|
||
Arguments:
|
||
|
||
SignalDone - Supplies a pointer to a variable that is cleared when the
|
||
requested operation has been performed.
|
||
|
||
Invalid - Supplies a bollean value that determines whether the virtual
|
||
address is invalid.
|
||
|
||
Number - Supplies the number of TB entries to flush.
|
||
|
||
Virtual - Supplies a pointer to an array of virtual addresses that
|
||
are within the pages whose translation buffer entries are to be
|
||
flushed.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
ULONG Index;
|
||
PVOID VirtualAddress[FLUSH_MULTIPLE_MAXIMUM];
|
||
|
||
//
|
||
// Capture the virtual addresses that are to be flushed from the TB
|
||
// on the current processor and signal pack done.
|
||
//
|
||
|
||
for (Index = 0; Index < (ULONG) Number; Index += 1) {
|
||
VirtualAddress[Index] = ((PVOID *)(Virtual))[Index];
|
||
}
|
||
|
||
KiIpiSignalPacketDone(SignalDone);
|
||
|
||
//
|
||
// Flush the specified virtual address for the TB on the current
|
||
// processor.
|
||
//
|
||
|
||
for (Index = 0; Index < (ULONG) Number; Index += 1) {
|
||
KiFlushSingleTb((BOOLEAN)Invalid, VirtualAddress [Index]);
|
||
}
|
||
}
|
||
|
||
HARDWARE_PTE
|
||
KeFlushSingleTb (
|
||
IN PVOID Virtual,
|
||
IN BOOLEAN Invalid,
|
||
IN BOOLEAN AllProcessors,
|
||
IN PHARDWARE_PTE PtePointer,
|
||
IN HARDWARE_PTE PteValue
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function flushes a single entry from translation buffer (TB) on all
|
||
processors that are currently running threads which are child of the current
|
||
process or flushes the entire translation buffer on all processors in the
|
||
host configuration.
|
||
|
||
Arguments:
|
||
|
||
Virtual - Supplies a virtual address that is within the page whose
|
||
translation buffer entry is to be flushed.
|
||
|
||
Invalid - Supplies a boolean value that specifies the reason for flushing
|
||
the translation buffer.
|
||
|
||
AllProcessors - Supplies a boolean value that determines which translation
|
||
buffers are to be flushed.
|
||
|
||
PtePointer - Address of Pte to update with new value.
|
||
|
||
PteValue - New value to put in the Pte. Will simply be assigned to
|
||
*PtePointer, in a fashion correct for the hardware.
|
||
|
||
Return Value:
|
||
|
||
Returns the contents of the PtePointer before the new value
|
||
is stored.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
KIRQL OldIrql;
|
||
PKPRCB Prcb;
|
||
PKPROCESS Process;
|
||
HARDWARE_PTE OldPteValue;
|
||
KAFFINITY TargetProcessors;
|
||
|
||
//
|
||
// Compute target set of processors.
|
||
//
|
||
|
||
if (AllProcessors != FALSE) {
|
||
OldIrql = KeRaiseIrqlToSynchLevel();
|
||
Prcb = KeGetCurrentPrcb();
|
||
TargetProcessors = KeActiveProcessors;
|
||
|
||
} else {
|
||
KiLockContextSwap(&OldIrql);
|
||
Prcb = KeGetCurrentPrcb();
|
||
Process = Prcb->CurrentThread->ApcState.Process;
|
||
TargetProcessors = Process->ActiveProcessors;
|
||
}
|
||
|
||
TargetProcessors &= ~Prcb->SetMember;
|
||
|
||
//
|
||
// Capture the previous contents of the page table entry and set the
|
||
// page table entry to the new value.
|
||
//
|
||
|
||
KI_SWAP_PTE(PtePointer, PteValue, OldPteValue);
|
||
|
||
//
|
||
// If any target processors are specified, then send a flush single
|
||
// packet to the target set of processors.
|
||
//
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiSendPacket(TargetProcessors,
|
||
KiFlushTargetSingleTb,
|
||
(PVOID)Invalid,
|
||
(PVOID)Virtual,
|
||
NULL);
|
||
|
||
IPI_INSTRUMENT_COUNT(Prcb->Number, FlushSingleTb);
|
||
}
|
||
|
||
|
||
//
|
||
// Flush the specified entry from the TB on the current processor.
|
||
//
|
||
|
||
KiFlushSingleTb(Invalid, Virtual);
|
||
|
||
//
|
||
// Wait until all target processors have finished and complete packet.
|
||
//
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiStallOnPacketTargets(TargetProcessors);
|
||
}
|
||
|
||
//
|
||
// Release the context swap lock.
|
||
//
|
||
|
||
if (AllProcessors != FALSE) {
|
||
KeLowerIrql(OldIrql);
|
||
|
||
} else {
|
||
KiUnlockContextSwap(OldIrql);
|
||
}
|
||
|
||
return(OldPteValue);
|
||
}
|
||
|
||
VOID
|
||
KiFlushTargetSingleTb (
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Invalid,
|
||
IN PVOID VirtualAddress,
|
||
IN PVOID Parameter3
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This is the target function for flushing a single TB entry.
|
||
|
||
Arguments:
|
||
|
||
SignalDone Supplies a pointer to a variable that is cleared when the
|
||
requested operation has been performed.
|
||
|
||
Invalid - Supplies a bollean value that determines whether the virtual
|
||
address is invalid.
|
||
|
||
Virtual - Supplies a virtual address that is within the page whose
|
||
translation buffer entry is to be flushed.
|
||
|
||
Parameter3 - Not used.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
//
|
||
// Flush a single entry from the TB on the current processor.
|
||
//
|
||
|
||
KiIpiSignalPacketDone(SignalDone);
|
||
KiFlushSingleTb((BOOLEAN)Invalid, (PVOID)VirtualAddress);
|
||
}
|
||
|
||
HARDWARE_PTE
|
||
KiFlushSingleTbSynchronous (
|
||
IN PVOID Virtual,
|
||
IN BOOLEAN Invalid,
|
||
IN BOOLEAN AllProcessors,
|
||
IN PHARDWARE_PTE PtePointer,
|
||
IN HARDWARE_PTE PteValue
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function is a slow synchronous version of KeFlushSingleTb. We need
|
||
this function as many P6's don't actually know how to deal with PTEs in
|
||
an MP safe manner.
|
||
|
||
Arguments:
|
||
|
||
See KeFlushSingleTb
|
||
|
||
Return Value:
|
||
|
||
See KeFlushSingleTb
|
||
|
||
--*/
|
||
{
|
||
|
||
KIRQL OldIrql;
|
||
PKPRCB Prcb;
|
||
PKPROCESS Process;
|
||
HARDWARE_PTE OldPteValue;
|
||
KAFFINITY TargetProcessors;
|
||
|
||
//
|
||
// Synchronize will all other single flush calls (and other
|
||
// IPIs which use reverse stalls)
|
||
//
|
||
|
||
KiLockContextSwap(&OldIrql);
|
||
|
||
//
|
||
// Compute target set of processors.
|
||
//
|
||
|
||
Prcb = KeGetCurrentPrcb();
|
||
if (AllProcessors != FALSE) {
|
||
TargetProcessors = KeActiveProcessors;
|
||
} else {
|
||
Process = Prcb->CurrentThread->ApcState.Process;
|
||
TargetProcessors = Process->ActiveProcessors;
|
||
}
|
||
|
||
TargetProcessors &= ~Prcb->SetMember;
|
||
|
||
//
|
||
// If any target processors are specified, then send a flush single
|
||
// packet to the target set of processors.
|
||
//
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiSendSynchronousPacket(Prcb,
|
||
TargetProcessors,
|
||
KiFlushTargetSingleTbSynchronous,
|
||
(PVOID)Invalid,
|
||
(PVOID)Virtual,
|
||
(PVOID)&Prcb->ReverseStall
|
||
);
|
||
|
||
IPI_INSTRUMENT_COUNT(Prcb->Number, FlushSingleTb);
|
||
|
||
//
|
||
// Wait for the target processors to stall
|
||
//
|
||
|
||
KiIpiStallOnPacketTargets(TargetProcessors);
|
||
|
||
//
|
||
// Capture the previous contents of the page table entry and set the
|
||
// page table entry to the new value.
|
||
//
|
||
|
||
KI_SWAP_PTE(PtePointer, PteValue, OldPteValue);
|
||
|
||
//
|
||
// Notify all prcessors it's time to go
|
||
//
|
||
|
||
Prcb->ReverseStall += 1;
|
||
|
||
} else {
|
||
|
||
//
|
||
// Capture the previous contents of the page table entry and set the
|
||
// page table entry to the new value.
|
||
//
|
||
|
||
KI_SWAP_PTE(PtePointer, PteValue, OldPteValue);
|
||
}
|
||
|
||
//
|
||
// Flush the specified entry from the TB on the current processor.
|
||
//
|
||
|
||
KiFlushSingleTb(Invalid, Virtual);
|
||
|
||
//
|
||
// Done
|
||
//
|
||
|
||
KiUnlockContextSwap(OldIrql);
|
||
return(OldPteValue);
|
||
}
|
||
|
||
VOID
|
||
KiFlushTargetSingleTbSynchronous (
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Invalid,
|
||
IN PVOID VirtualAddress,
|
||
IN PVOID Proceed
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This is the target function for flushing a single TB entry synchronously.
|
||
|
||
Arguments:
|
||
|
||
SignalDone Supplies a pointer to a variable that is cleared when the
|
||
requested operation has been performed.
|
||
|
||
Invalid - Supplies a bollean value that determines whether the virtual
|
||
address is invalid.
|
||
|
||
Virtual - Supplies a virtual address that is within the page whose
|
||
translation buffer entry is to be flushed.
|
||
|
||
Parameter3 - Not used.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
//
|
||
// Flush a single entry from the TB on the current processor.
|
||
//
|
||
|
||
KiIpiSignalPacketDoneAndStall(SignalDone, Proceed);
|
||
KiFlushSingleTb((BOOLEAN)Invalid, (PVOID)VirtualAddress);
|
||
}
|
||
|
||
|
||
VOID
|
||
Ki386UseSynchronousTbFlush (
|
||
IN volatile PLONG Number
|
||
)
|
||
{
|
||
PKPRCB Prcb;
|
||
volatile PUCHAR Patch;
|
||
|
||
Prcb = KeGetCurrentPrcb();
|
||
Patch = (PUCHAR) KeFlushSingleTb;
|
||
|
||
//
|
||
// Signal we're here and wait for others
|
||
//
|
||
|
||
InterlockedDecrement (Number);
|
||
while (*Number) ;
|
||
|
||
//
|
||
// If this is processor 0 apply the patch
|
||
//
|
||
|
||
if (Prcb->Number == 0) {
|
||
*((PULONG) &Patch[1]) = ((ULONG) &KiFlushSingleTbSynchronous) - ((ULONG) Patch) - 5;
|
||
Patch[0] = 0xe9;
|
||
}
|
||
|
||
//
|
||
// Wait for processor 0 to complete installation of handler
|
||
//
|
||
|
||
while (Patch[0] != 0xe9) ;
|
||
}
|
||
|
||
#endif
|