able/windows-nt
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
windows-nt/Source/XPSP1/NT/base/ntos/ke/ia64/flush.c

1083 lines
22 KiB
C
Raw Normal View History

Add source files
2020-09-26 03:20:57 -05:00
/*++
Module Name:
flush.c
Abstract:
This module implements IA64 machine dependent kernel functions to flush
the data and instruction caches and to flush I/O buffers.
Author:
07-Mar-1996
Bernard Lint
M. Jayakumar (Muthurajan.Jayakumar@intel.com)
Environment:
Kernel mode only.
Revision History:
--*/
#include "ki.h"
#include "kxia64.h"
//
// PROBE_VISIBILITY_PAL_SUPPORT flag is one time write (RESET) only and multiple time read
// only flag. It is used to check to see if the processor needs PAL_SUPPORT for VISIBILITY // in prefetches. Once the check is made, this flag optimizes such that further checks are // eliminated.
//
ULONG ProbePalVisibilitySupport=1;
ULONG NeedPalVisibilitySupport=1;
extern KSPIN_LOCK KiCacheFlushLock;
//
// Define forward referenced prototyes.
//
VOID
KiSweepDcacheTarget (
IN PULONG SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
);
VOID
KiSweepIcacheTarget (
IN PULONG SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
);
VOID
KiFlushIoBuffersTarget (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Mdl,
IN PVOID ReadOperation,
IN PVOID DmaOperation
);
VOID
KiSyncCacheTarget(
IN PKIPI_CONTEXT SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
);
ULONG_PTR
KiSyncMC_DrainTarget(
);
ULONG_PTR
KiSyncMC_Drain(
IN BOOLEAN AllProcessors,
IN PVOID BaseAddress,
IN ULONG Length
);
ULONG_PTR
KiSyncPrefetchVisibleTarget(
);
ULONG_PTR
KiSyncPrefetchVisible (
IN BOOLEAN AllProcessors,
IN PVOID BaseAddress,
IN ULONG Length
);
VOID
KiSyncCacheTarget (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
)
/*++
Routine Description:
This function synchronizes the I-fetch pipeline. Typically this routine will be
executed by every processor in the system in response to an IPI after the cache
is flushed. Each processor executing RFI while leaving the IPI produces the
serialization effect that is required after isync to make sure that further
instruction prefetches wait till the ISYNC completes.
Arguements:
SignalDone Supplies a pointer to a variable that is cleared when the
requested operation has been performed.
Parameter1 - Parameter3 - Not used.
Return Value:
Nothing.
--*/
{
#if !defined(NT_UP)
__synci();
KiIpiSignalPacketDone(SignalDone);
#endif
return;
}
VOID
KeSweepIcache (
IN BOOLEAN AllProcessors
)
/*++
Routine Description:
This function flushes the instruction cache on all processors that are
currently running threads which are children of the current process or
flushes the instruction cache on all processors in the host configuration.
N.B. Although PowerPC maintains cache coherency across processors, we
use the flash invalidate function (h/w) for I-Cache sweeps which doesn't
maintain coherency so we still do the MP I-Cache flush in s/w. plj.
Arguments:
AllProcessors - Supplies a boolean value that determines which instruction
caches are flushed.
Return Value:
None.
--*/
{
KIRQL OldIrql;
KAFFINITY TargetProcessors;
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
#if !defined(NT_UP)
//
// Acquire cache flush spinlock
// Cache flush is not MP safe yet
//
KeAcquireSpinLock(&KiCacheFlushLock, &OldIrql);
#endif
HalSweepIcache();
#if !defined(NT_UP)
//
// Compute the set of target processors and send the sweep parameters
// to the target processors, if any, for execution.
//
TargetProcessors = KeActiveProcessors & PCR->NotMember;
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiSweepIcacheTarget,
NULL,
NULL,
NULL);
}
//
// Wait until all target processors have finished sweeping their
// instruction caches.
//
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
//
// Lower IRQL to its previous level and return.
//
KeReleaseSpinLock(&KiCacheFlushLock, OldIrql);
#endif
return;
}
VOID
KiSweepIcacheTarget (
IN PULONG SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
)
/*++
Routine Description:
This is the target function for sweeping the instruction cache on
target processors.
Arguments:
SignalDone Supplies a pointer to a variable that is cleared when the
requested operation has been performed.
Parameter1 - Parameter3 - Not used.
Return Value:
None.
--*/
{
//
// Sweep the instruction cache on the current processor and clear
// the sweep instruction cache packet address to signal the source
// to continue.
//
#if !defined(NT_UP)
HalSweepIcache();
KiIpiSignalPacketDone(SignalDone);
#endif
return;
}
VOID
KeSweepDcache (
IN BOOLEAN AllProcessors
)
/*++
Routine Description:
This function flushes the data cache on all processors that are currently
running threads which are children of the current process or flushes the
data cache on all processors in the host configuration.
N.B. PowerPC maintains cache coherency across processors however
in this routine, the range of addresses being flushed is unknown
so we must still broadcast the request to the other processors.
Arguments:
AllProcessors - Supplies a boolean value that determines which data
caches are flushed.
Return Value:
None.
--*/
{
KIRQL OldIrql;
KAFFINITY TargetProcessors;
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
#if !defined(NT_UP)
//
// Acquire cache flush spinlock
// Cache flush is not MP safe yet
//
KeAcquireSpinLock(&KiCacheFlushLock, &OldIrql);
#endif
HalSweepDcache();
#if !defined(NT_UP)
//
// Compute the set of target processors and send the sweep parameters
// to the target processors, if any, for execution.
//
TargetProcessors = KeActiveProcessors & PCR->NotMember;
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiSweepDcacheTarget,
NULL,
NULL,
NULL);
}
//
// Wait until all target processors have finished sweeping their
// data caches.
//
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
//
// Lower IRQL to its previous level and return.
//
KeReleaseSpinLock(&KiCacheFlushLock, OldIrql);
#endif
return;
}
VOID
KiSweepDcacheTarget (
IN PULONG SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
)
/*++
Routine Description:
This is the target function for sweeping the data cache on target
processors.
Arguments:
SignalDone Supplies a pointer to a variable that is cleared when the
requested operation has been performed.
Parameter1 - Parameter3 - Not used.
Return Value:
None.
--*/
{
//
// Sweep the data cache on the current processor and clear the sweep
// data cache packet address to signal the source to continue.
//
#if !defined(NT_UP)
HalSweepDcache();
KiIpiSignalPacketDone(SignalDone);
#endif
return;
}
ULONG_PTR
KiSyncMC_DrainTarget(
)
/*++
Routine Description:
This is the target function for issuing PAL_MC_DRAIN to drain
prefetches, demand references and pending fc cache line evictions on the
target CPU it executes.
Argument:
None
Return Value:
Returns the status from the function HalCallPal
--*/
{
ULONG_PTR Status;
//
// Call HalCallPal to drain.
//
Status = HalCallPal(PAL_MC_DRAIN,
0,
0,
0,
0,
0,
0,
0);
ASSERT(Status == PAL_STATUS_SUCCESS);
return Status;
}
VOID
KeSweepCacheRange (
IN BOOLEAN AllProcessors,
IN PVOID BaseAddress,
IN ULONG Length
)
/*++
Routine Description:
This function is used to flush a range of virtual addresses from both the
instruction and data cache on all processors in the system.
Irrespective of the length of the range, it should not call SweepIcache
or SweepDcache. This is because SweepDcache will only sweep D cache and
not the I cache and Vice versa. Since the caller of KeSweepCacheRange assumes
both the caches are being swept, one cannot call SweepIcache or SweepDcache
in trying to optimize.
Arguments:
AllProcessors - Not used
BaseAddress - Supplies a pointer to the base of the range that is flushed.
Length - Supplies the length of the range that is flushed if the base
address is specified.
Return Value:
None.
--*/
{
KIRQL OldIrql;
KAFFINITY TargetProcessors;
//
// We will not raise IRQL to synchronization level so that we can allow
// a context switch in between Flush Cache. FC need not run in the same processor
// throughout. It can be context switched. So no binding is done to any processor.
//
//
HalSweepCacheRange(BaseAddress,Length);
ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
//
// Raise IRQL to synchronization level to prevent a context switch.
//
#if !defined(NT_UP)
OldIrql = KeRaiseIrqlToSynchLevel();
//
// Compute the set of target processors and send the sync parameters
// to the target processors, if any, for execution.
//
TargetProcessors = KeActiveProcessors & PCR->NotMember;
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiSyncCacheTarget,
NULL,
NULL,
NULL);
}
#endif
//
// Synchronize the Instruction Prefetch pipe in the local processor.
//
__synci();
__isrlz();
//
// Wait until all target processors have finished sweeping the their
// data cache.
//
#if !defined(NT_UP)
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
//
// Lower IRQL to its previous level and return.
//
KeLowerIrql(OldIrql);
#endif
return;
}
VOID
KeSweepIcacheRange (
IN BOOLEAN AllProcessors,
IN PVOID BaseAddress,
IN SIZE_T Length
)
/*++
Routine Description:
This function is used to flush a range of virtual addresses from the
primary instruction cache on all processors in the host configuration.
If the length of the range is greater than the size of the
instruction cache, then one can call HalSweepIcache which calls
SAL to flush the entire cache. Since SAL does not take care of MP
flushing, HalSweepIcache has to use IPI mechanism to execute SAL
flush from each processor. We need to weight the overhead of all these
versus using HalSweepIcacheRange and avoiding IPI mechanism since
HalSweepIcacheRange uses fc instruction and fc instruction takes care of MP.
Arguments:
AllProcessors - Not used
BaseAddress - Supplies a pointer to the base of the range that is flushed.
Length - Supplies the length of the range that is flushed if the base
address is specified.
Return Value:
None.
Note: For performance reason, we may update KeSweepIcacheRange to do the following:
if the range asked to sweep is very large, we may call KeSweepIcache to flush
the full cache.
--*/
{
KIRQL OldIrql;
KAFFINITY TargetProcessors;
//
// We will not raise IRQL to synchronization level so that we can allow
// a context switch in between Flush Cache. FC need not run in the same processor
// throughout. It can be context switched. So no binding is done to any processor.
//
//
HalSweepIcacheRange(BaseAddress,Length);
ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
//
// Raise IRQL to synchronization level to prevent a context switch.
//
#if !defined(NT_UP)
OldIrql = KeRaiseIrqlToSynchLevel();
//
// Compute the set of target processors and send the sync parameters
// to the target processors, if any, for execution.
//
TargetProcessors = KeActiveProcessors & PCR->NotMember;
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiSyncCacheTarget,
NULL,
NULL,
NULL);
}
#endif
//
// Synchronize the Instruction Prefetch pipe in the local processor.
//
__synci();
__isrlz();
//
// Wait until all target processors have finished sweeping the their
// data cache.
//
#if !defined(NT_UP)
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
//
// Lower IRQL to its previous level and return.
//
KeLowerIrql(OldIrql);
#endif
return;
}
VOID
KeSweepCurrentIcacheRange (
IN PVOID BaseAddress,
IN SIZE_T Length
)
/*++
Routine Description:
This function is used to flush a range of virtual addresses from the
primary instruction cache on the current processor.
This is used by the kernel debugger for flushing the i-cache after
modifying memory in case the instruction stream is changed.
To avoid calling SAL during phase 0 we use "fc" instead of the SAL cache
flush call.
Arguments:
BaseAddress - Supplies a pointer to the base of the range that is flushed.
Length - Supplies the length of the range that is flushed if the base
address is specified.
Return Value:
None.
--*/
{
KIRQL OldIrql;
KeRaiseIrql(HIGH_LEVEL, &OldIrql);
HalSweepIcacheRange(BaseAddress,Length);
//
// Synchronize the Instruction Prefetch pipe in the local processor.
//
__synci();
__isrlz();
KeLowerIrql(OldIrql);
return;
}
VOID
KeSweepDcacheRange (
IN BOOLEAN AllProcessors,
IN PVOID BaseAddress,
IN ULONG Length
)
/*++
Routine Description:
This function is used to flush a range of virtual addresses from the
primary data cache on all processors in the host configuration.
If the length of the range is greater than the size of the
data cache, then one can call HalSweepDcache which calls
SAL to flush the entire cache. Since SAL does not take care of MP
flushing, HalSweepDcache has to use IPI mechanism to execute SAL
flush from each processor. We need to weight the overhead of all these
versus using HalSweepDcacheRange and avoiding IPI mechanism since
HalSweepDcacheRange uses fc instruction and fc instruction takes care of MP.
Arguments:
AllProcessors - Not used
BaseAddress - Supplies a pointer to the base of the range that is flushed.
Length - Supplies the length of the range that is flushed if the base
address is specified.
Return Value:
None.
Note: For performance reason, we may update KeSweepDcacheRange to do the following:
if the range asked to sweep is very large, we may call KeSweepDcache to flush
the full cache.
--*/
{
KIRQL OldIrql;
KAFFINITY TargetProcessors;
//
// We will not raise IRQL to synchronization level so that we can allow
// a context switch in between Flush Cache. FC need not run in the same processor
// throughout. It can be context switched. So no binding is done to any processor.
//
//
HalSweepDcacheRange(BaseAddress,Length);
ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
//
// Raise IRQL to synchronization level to prevent a context switch.
//
#if !defined(NT_UP)
OldIrql = KeRaiseIrqlToSynchLevel();
//
// Compute the set of target processors and send the sync parameters
// to the target processors, if any, for execution.
//
TargetProcessors = KeActiveProcessors & PCR->NotMember;
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiSyncCacheTarget,
NULL,
NULL,
NULL);
}
#endif
//
// Synchronize the Instruction Prefetch pipe in the local processor.
//
__synci();
__isrlz();
//
// Wait until all target processors have finished sweeping the their
// data cache.
//
#if !defined(NT_UP)
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
//
// Lower IRQL to its previous level and return.
//
KeLowerIrql(OldIrql);
#endif
return;
}
ULONG_PTR
KiSyncMC_Drain (
IN BOOLEAN AllProcessors,
IN PVOID BaseAddress,
IN ULONG Length
)
/*++
Routine Description:
KiSyncMC_Drain issues PAL_MC_DRAIN to drain either prefetches, demand references
or pending fc cache line evictions to all the processors in the system.
DrainTypePointer points to the variable, DrainType, which determines the type of
drain to be performed. This is typically used when changing the memory attribute
from WB to UC.
Arguments:
AllProcessors - All processors in the system.
BaseAddress - Supplies a pointer to the base of the range that is to be drained.
Length - Supplies the length of the range that is drained for the base
address specified.
Return Value:
Note: This is used when changing attributes of WB pages to UC pages.
--*/
{
ULONG_PTR Status;
//
// KiIpiGenericCall returns ULONG_PTR as the function value of the specified function
//
Status = (KiIpiGenericCall (
(PKIPI_BROADCAST_WORKER)KiSyncMC_DrainTarget,
(ULONG_PTR)NULL)
);
ASSERT(Status == PAL_STATUS_SUCCESS);
return Status;
}
ULONG_PTR
KiSyncPrefetchVisibleTarget(
)
/*++
Routine Description:
This is the target function for issuing PAL_PREFETCH VISIBILITY
on the target CPU it executes.
Argument:
Not used.
Return Value:
Returns the status from the function HalCallPal
--*/
{
ULONG_PTR Status;
//
// Call HalCallPal to drain.
//
Status = HalCallPal(PAL_PREFETCH_VISIBILITY,
0,
0,
0,
0,
0,
0,
0);
ASSERT(Status != PAL_STATUS_ERROR);
return Status;
}
ULONG_PTR
KiSyncPrefetchVisible (
IN BOOLEAN AllProcessors,
IN PVOID BaseAddress,
IN ULONG Length
)
/*++
Routine Description:
KiSyncPrefetchVisible issues PAL_PREFETCH_VISIBILITY to cause the processor to make
all pending prefetches visible to subsequent fc instructions; or does nothing, on
processor implementations which does not require PAL support for disabling prefetch
in the architectural sequence. On processors that require PAL support for this
sequence, the actions performed by this procedure may include any or all
of the following (or none, as long as the processor guarantees that
prefetches that were issued prior to this call are not resident in the
processor's caches after the architected sequence is complete.
This is typically used when changing the memory attribute from WB to UC.
Arguments:
AllProcessors - All processors in the system.
BaseAddress - Supplies a pointer to the base of the range that is to be drained.
Length - Supplies the length of the range that is drained for the base
address specified.
Return Value:
Status of the PAL CALL
0 Success
1 Call not needed
-3 Error returned
Note: This is used when changing attributes of WB pages to UC pages.
--*/
{
ULONG_PTR Status;
switch (ProbePalVisibilitySupport) {
case 0:
if (NeedPalVisibilitySupport == 0)
Status = PAL_STATUS_SUPPORT_NOT_NEEDED;
else {
Status = (KiIpiGenericCall (
(PKIPI_BROADCAST_WORKER)KiSyncPrefetchVisibleTarget,
(ULONG_PTR)NULL)
);
}
break;
case 1:
Status = KiSyncPrefetchVisibleTarget();
ASSERT(Status != PAL_STATUS_ERROR);
ProbePalVisibilitySupport = 0;
if (Status == PAL_STATUS_SUPPORT_NOT_NEEDED) {
NeedPalVisibilitySupport = 0;
Status = PAL_STATUS_SUPPORT_NOT_NEEDED;
} else {
Status = (KiIpiGenericCall (
(PKIPI_BROADCAST_WORKER)KiSyncPrefetchVisibleTarget,
(ULONG_PTR)NULL)
);
}
break;
default:
Status = PAL_STATUS_ERROR;
break;
}
ASSERT(Status != PAL_STATUS_ERROR);
return Status;
}
VOID
KeSweepCacheRangeWithDrain (
IN BOOLEAN AllProcessors,
IN PVOID BaseAddress,
IN ULONG Length
)
/*++
Routine Description:
This function is used to drain prefetches,demand references followed by flushing
the cache followed by draining pending fc cache line evictions to a specified range
address in all processors in the system.
Arguments:
AllProcessors - All processors in the system.
BaseAddress - Supplies a pointer to the base of the range that is flushed and drained.
Length - Supplies the length of the range that is flushed and drained for the base
address is specified.
Return Value:
None.
Note: This is used when changing attributes of WB pages to UC pages.
--*/
{
ULONG_PTR Status;
Status = KiSyncPrefetchVisible(
AllProcessors,
BaseAddress,
Length
);
ASSERT(Status != PAL_STATUS_ERROR);
KeSweepCacheRange (
AllProcessors,
BaseAddress,
Length
);
Status = KiSyncMC_Drain (
AllProcessors,
BaseAddress,
Length
);
ASSERT(Status == PAL_STATUS_SUCCESS);
return;
}
Reference in a new issue Copy permalink