able/windows-nt
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
windows-nt/Source/XPSP1/NT/base/ntos/ke/i386/flushtb.c

791 lines
17 KiB
C
Raw Normal View History

Add source files
2020-09-26 03:20:57 -05:00
/*++
Copyright (c) 1989 Microsoft Corporation
Copyright (c) 1990 Microsoft Corporation
Module Name:
tbflush.c
Abstract:
This module implements machine dependent functions to flush
the translation buffers in an Intel x86 system.
N.B. This module contains only MP versions of the TB flush routines.
The UP versions are macros in ke.h
KeFlushEntireTb remains a routine for the UP system since it is
exported from the kernel for backwards compatibility.
Author:
David N. Cutler (davec) 13-May-1989
Environment:
Kernel mode only.
Revision History:
Shie-Lin Tzong (shielint) 30-Aug-1990
Implement MP version of KeFlushSingleTb and KeFlushEntireTb.
--*/
#include "ki.h"
VOID
KiFlushTargetEntireTb (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Invalid,
IN PVOID Parameter2,
IN PVOID Parameter3
);
VOID
KiFlushTargetMultipleTb (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
);
VOID
KiFlushTargetSingleTb (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
);
HARDWARE_PTE
KiFlushSingleTbSynchronous (
IN PVOID Virtual,
IN BOOLEAN Invalid,
IN BOOLEAN AllProcessors,
IN PHARDWARE_PTE PtePointer,
IN HARDWARE_PTE PteValue
);
VOID
KiFlushTargetSingleTbSynchronous (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
);
VOID
Ki386UseSynchronousTbFlush (
IN volatile PLONG Number
);
#ifdef ALLOC_PRAGMA
#pragma alloc_text(INIT,Ki386UseSynchronousTbFlush)
#endif
#if !defined(NT_UP)
VOID
KeFlushEntireTb (
IN BOOLEAN Invalid,
IN BOOLEAN AllProcessors
)
/*++
Routine Description:
This function flushes the entire translation buffer (TB) on all processors
that are currently running threads which are child of the current process
or flushes the entire translation buffer on all processors in the host
configuration.
Arguments:
Invalid - Supplies a boolean value that specifies the reason for flushing
the translation buffer.
AllProcessors - Supplies a boolean value that determines which translation
buffers are to be flushed.
Return Value:
None.
--*/
{
KAFFINITY EntireSet;
KIRQL OldIrql;
PKPRCB Prcb;
PKPROCESS Process;
KAFFINITY TargetProcessors;
//
// Compute the target set of processors, disable context switching,
// and send the flush entire parameters to the target processors,
// if any, for execution.
//
if (AllProcessors != FALSE) {
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
TargetProcessors = KeActiveProcessors;
} else {
KiLockContextSwap(&OldIrql);
Prcb = KeGetCurrentPrcb();
Process = Prcb->CurrentThread->ApcState.Process;
TargetProcessors = Process->ActiveProcessors;
}
EntireSet = KeActiveProcessors & ~Prcb->SetMember;
TargetProcessors &= ~Prcb->SetMember;
//
// If the target set of processors is equal to the full set of processors,
// then set the TB flush time stamp busy.
//
if (TargetProcessors == EntireSet) {
KiSetTbFlushTimeStampBusy();
}
//
// Send packet to target processors.
//
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiFlushTargetEntireTb,
NULL,
NULL,
NULL);
IPI_INSTRUMENT_COUNT (Prcb->Number, FlushEntireTb);
}
//
// Flush TB on current processor.
//
KeFlushCurrentTb();
//
// Wait until all target processors have finished and complete packet.
//
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
//
// If the target set of processors is equal to the full set of processors,
// then clear the TB flush time stamp busy.
//
if (TargetProcessors == EntireSet) {
KiClearTbFlushTimeStampBusy();
}
//
// Lower IRQL and unlock as appropriate.
//
if (AllProcessors != FALSE) {
KeLowerIrql(OldIrql);
} else {
KiUnlockContextSwap(OldIrql);
}
return;
}
VOID
KiFlushTargetEntireTb (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
)
/*++
Routine Description:
This is the target function for flushing the entire TB.
Arguments:
SignalDone - Supplies a pointer to a variable that is cleared when the
requested operation has been performed.
Parameter1 - Parameter3 - Not used.
Return Value:
None.
--*/
{
//
// Flush the entire TB on the current processor.
//
KiIpiSignalPacketDone(SignalDone);
KeFlushCurrentTb();
return;
}
VOID
KeFlushMultipleTb (
IN ULONG Number,
IN PVOID *Virtual,
IN BOOLEAN Invalid,
IN BOOLEAN AllProcessors,
IN PHARDWARE_PTE *PtePointer OPTIONAL,
IN HARDWARE_PTE PteValue
)
/*++
Routine Description:
This function flushes multiple entries from the translation buffer
on all processors that are currently running threads which are
children of the current process or flushes a multiple entries from
the translation buffer on all processors in the host configuration.
Arguments:
Number - Supplies the number of TB entries to flush.
Virtual - Supplies a pointer to an array of virtual addresses that
are within the pages whose translation buffer entries are to be
flushed.
Invalid - Supplies a boolean value that specifies the reason for
flushing the translation buffer.
AllProcessors - Supplies a boolean value that determines which
translation buffers are to be flushed.
PtePointer - Supplies an optional pointer to an array of pointers to
page table entries that receive the specified page table entry
value.
PteValue - Supplies the the new page table entry value.
Return Value:
The previous contents of the specified page table entry is returned
as the function value.
--*/
{
ULONG Index;
KIRQL OldIrql;
PKPRCB Prcb;
PKPROCESS Process;
KAFFINITY TargetProcessors;
//
// Compute target set of processors.
//
if (AllProcessors != FALSE) {
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
TargetProcessors = KeActiveProcessors;
} else {
KiLockContextSwap(&OldIrql);
Prcb = KeGetCurrentPrcb();
Process = Prcb->CurrentThread->ApcState.Process;
TargetProcessors = Process->ActiveProcessors;
}
TargetProcessors &= ~Prcb->SetMember;
//
// If a page table entry address array is specified, then set the
// specified page table entries to the specific value.
//
if (ARGUMENT_PRESENT(PtePointer)) {
for (Index = 0; Index < Number; Index += 1) {
KI_FILL_PTE(PtePointer[Index], PteValue);
}
}
//
// If any target processors are specified, then send a flush multiple
// packet to the target set of processors.
//
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiFlushTargetMultipleTb,
(PVOID)Invalid,
(PVOID)Number,
(PVOID)Virtual);
IPI_INSTRUMENT_COUNT (Prcb->Number, FlushMultipleTb);
}
//
// Flush the specified entries from the TB on the current processor.
//
for (Index = 0; Index < Number; Index += 1) {
KiFlushSingleTb(Invalid, Virtual[Index]);
}
//
// Wait until all target processors have finished and complete packet.
//
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
//
// Release the context swap lock.
//
if (AllProcessors != FALSE) {
KeLowerIrql(OldIrql);
} else {
KiUnlockContextSwap(OldIrql);
}
return;
}
VOID
KiFlushTargetMultipleTb (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Invalid,
IN PVOID Number,
IN PVOID Virtual
)
/*++
Routine Description:
This is the target function for flushing multiple TB entries.
Arguments:
SignalDone - Supplies a pointer to a variable that is cleared when the
requested operation has been performed.
Invalid - Supplies a bollean value that determines whether the virtual
address is invalid.
Number - Supplies the number of TB entries to flush.
Virtual - Supplies a pointer to an array of virtual addresses that
are within the pages whose translation buffer entries are to be
flushed.
Return Value:
None.
--*/
{
ULONG Index;
PVOID VirtualAddress[FLUSH_MULTIPLE_MAXIMUM];
//
// Capture the virtual addresses that are to be flushed from the TB
// on the current processor and signal pack done.
//
for (Index = 0; Index < (ULONG) Number; Index += 1) {
VirtualAddress[Index] = ((PVOID *)(Virtual))[Index];
}
KiIpiSignalPacketDone(SignalDone);
//
// Flush the specified virtual address for the TB on the current
// processor.
//
for (Index = 0; Index < (ULONG) Number; Index += 1) {
KiFlushSingleTb((BOOLEAN)Invalid, VirtualAddress [Index]);
}
}
HARDWARE_PTE
KeFlushSingleTb (
IN PVOID Virtual,
IN BOOLEAN Invalid,
IN BOOLEAN AllProcessors,
IN PHARDWARE_PTE PtePointer,
IN HARDWARE_PTE PteValue
)
/*++
Routine Description:
This function flushes a single entry from translation buffer (TB) on all
processors that are currently running threads which are child of the current
process or flushes the entire translation buffer on all processors in the
host configuration.
Arguments:
Virtual - Supplies a virtual address that is within the page whose
translation buffer entry is to be flushed.
Invalid - Supplies a boolean value that specifies the reason for flushing
the translation buffer.
AllProcessors - Supplies a boolean value that determines which translation
buffers are to be flushed.
PtePointer - Address of Pte to update with new value.
PteValue - New value to put in the Pte. Will simply be assigned to
*PtePointer, in a fashion correct for the hardware.
Return Value:
Returns the contents of the PtePointer before the new value
is stored.
--*/
{
KIRQL OldIrql;
PKPRCB Prcb;
PKPROCESS Process;
HARDWARE_PTE OldPteValue;
KAFFINITY TargetProcessors;
//
// Compute target set of processors.
//
if (AllProcessors != FALSE) {
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
TargetProcessors = KeActiveProcessors;
} else {
KiLockContextSwap(&OldIrql);
Prcb = KeGetCurrentPrcb();
Process = Prcb->CurrentThread->ApcState.Process;
TargetProcessors = Process->ActiveProcessors;
}
TargetProcessors &= ~Prcb->SetMember;
//
// Capture the previous contents of the page table entry and set the
// page table entry to the new value.
//
KI_SWAP_PTE(PtePointer, PteValue, OldPteValue);
//
// If any target processors are specified, then send a flush single
// packet to the target set of processors.
//
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiFlushTargetSingleTb,
(PVOID)Invalid,
(PVOID)Virtual,
NULL);
IPI_INSTRUMENT_COUNT(Prcb->Number, FlushSingleTb);
}
//
// Flush the specified entry from the TB on the current processor.
//
KiFlushSingleTb(Invalid, Virtual);
//
// Wait until all target processors have finished and complete packet.
//
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
//
// Release the context swap lock.
//
if (AllProcessors != FALSE) {
KeLowerIrql(OldIrql);
} else {
KiUnlockContextSwap(OldIrql);
}
return(OldPteValue);
}
VOID
KiFlushTargetSingleTb (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Invalid,
IN PVOID VirtualAddress,
IN PVOID Parameter3
)
/*++
Routine Description:
This is the target function for flushing a single TB entry.
Arguments:
SignalDone Supplies a pointer to a variable that is cleared when the
requested operation has been performed.
Invalid - Supplies a bollean value that determines whether the virtual
address is invalid.
Virtual - Supplies a virtual address that is within the page whose
translation buffer entry is to be flushed.
Parameter3 - Not used.
Return Value:
None.
--*/
{
//
// Flush a single entry from the TB on the current processor.
//
KiIpiSignalPacketDone(SignalDone);
KiFlushSingleTb((BOOLEAN)Invalid, (PVOID)VirtualAddress);
}
HARDWARE_PTE
KiFlushSingleTbSynchronous (
IN PVOID Virtual,
IN BOOLEAN Invalid,
IN BOOLEAN AllProcessors,
IN PHARDWARE_PTE PtePointer,
IN HARDWARE_PTE PteValue
)
/*++
Routine Description:
This function is a slow synchronous version of KeFlushSingleTb. We need
this function as many P6's don't actually know how to deal with PTEs in
an MP safe manner.
Arguments:
See KeFlushSingleTb
Return Value:
See KeFlushSingleTb
--*/
{
KIRQL OldIrql;
PKPRCB Prcb;
PKPROCESS Process;
HARDWARE_PTE OldPteValue;
KAFFINITY TargetProcessors;
//
// Synchronize will all other single flush calls (and other
// IPIs which use reverse stalls)
//
KiLockContextSwap(&OldIrql);
//
// Compute target set of processors.
//
Prcb = KeGetCurrentPrcb();
if (AllProcessors != FALSE) {
TargetProcessors = KeActiveProcessors;
} else {
Process = Prcb->CurrentThread->ApcState.Process;
TargetProcessors = Process->ActiveProcessors;
}
TargetProcessors &= ~Prcb->SetMember;
//
// If any target processors are specified, then send a flush single
// packet to the target set of processors.
//
if (TargetProcessors != 0) {
KiIpiSendSynchronousPacket(Prcb,
TargetProcessors,
KiFlushTargetSingleTbSynchronous,
(PVOID)Invalid,
(PVOID)Virtual,
(PVOID)&Prcb->ReverseStall
);
IPI_INSTRUMENT_COUNT(Prcb->Number, FlushSingleTb);
//
// Wait for the target processors to stall
//
KiIpiStallOnPacketTargets(TargetProcessors);
//
// Capture the previous contents of the page table entry and set the
// page table entry to the new value.
//
KI_SWAP_PTE(PtePointer, PteValue, OldPteValue);
//
// Notify all prcessors it's time to go
//
Prcb->ReverseStall += 1;
} else {
//
// Capture the previous contents of the page table entry and set the
// page table entry to the new value.
//
KI_SWAP_PTE(PtePointer, PteValue, OldPteValue);
}
//
// Flush the specified entry from the TB on the current processor.
//
KiFlushSingleTb(Invalid, Virtual);
//
// Done
//
KiUnlockContextSwap(OldIrql);
return(OldPteValue);
}
VOID
KiFlushTargetSingleTbSynchronous (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Invalid,
IN PVOID VirtualAddress,
IN PVOID Proceed
)
/*++
Routine Description:
This is the target function for flushing a single TB entry synchronously.
Arguments:
SignalDone Supplies a pointer to a variable that is cleared when the
requested operation has been performed.
Invalid - Supplies a bollean value that determines whether the virtual
address is invalid.
Virtual - Supplies a virtual address that is within the page whose
translation buffer entry is to be flushed.
Parameter3 - Not used.
Return Value:
None.
--*/
{
//
// Flush a single entry from the TB on the current processor.
//
KiIpiSignalPacketDoneAndStall(SignalDone, Proceed);
KiFlushSingleTb((BOOLEAN)Invalid, (PVOID)VirtualAddress);
}
VOID
Ki386UseSynchronousTbFlush (
IN volatile PLONG Number
)
{
PKPRCB Prcb;
volatile PUCHAR Patch;
Prcb = KeGetCurrentPrcb();
Patch = (PUCHAR) KeFlushSingleTb;
//
// Signal we're here and wait for others
//
InterlockedDecrement (Number);
while (*Number) ;
//
// If this is processor 0 apply the patch
//
if (Prcb->Number == 0) {
*((PULONG) &Patch[1]) = ((ULONG) &KiFlushSingleTbSynchronous) - ((ULONG) Patch) - 5;
Patch[0] = 0xe9;
}
//
// Wait for processor 0 to complete installation of handler
//
while (Patch[0] != 0xe9) ;
}
#endif
Reference in a new issue Copy permalink