1083 lines
22 KiB
C
1083 lines
22 KiB
C
|
||
/*++
|
||
|
||
Module Name:
|
||
|
||
flush.c
|
||
|
||
Abstract:
|
||
|
||
This module implements IA64 machine dependent kernel functions to flush
|
||
the data and instruction caches and to flush I/O buffers.
|
||
|
||
Author:
|
||
|
||
07-Mar-1996
|
||
|
||
Bernard Lint
|
||
M. Jayakumar (Muthurajan.Jayakumar@intel.com)
|
||
|
||
|
||
Environment:
|
||
|
||
Kernel mode only.
|
||
|
||
Revision History:
|
||
|
||
--*/
|
||
|
||
#include "ki.h"
|
||
#include "kxia64.h"
|
||
|
||
//
|
||
// PROBE_VISIBILITY_PAL_SUPPORT flag is one time write (RESET) only and multiple time read
|
||
// only flag. It is used to check to see if the processor needs PAL_SUPPORT for VISIBILITY // in prefetches. Once the check is made, this flag optimizes such that further checks are // eliminated.
|
||
//
|
||
|
||
ULONG ProbePalVisibilitySupport=1;
|
||
ULONG NeedPalVisibilitySupport=1;
|
||
extern KSPIN_LOCK KiCacheFlushLock;
|
||
//
|
||
// Define forward referenced prototyes.
|
||
//
|
||
|
||
VOID
|
||
KiSweepDcacheTarget (
|
||
IN PULONG SignalDone,
|
||
IN PVOID Parameter1,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
);
|
||
|
||
VOID
|
||
KiSweepIcacheTarget (
|
||
IN PULONG SignalDone,
|
||
IN PVOID Parameter1,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
);
|
||
|
||
VOID
|
||
KiFlushIoBuffersTarget (
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Mdl,
|
||
IN PVOID ReadOperation,
|
||
IN PVOID DmaOperation
|
||
);
|
||
|
||
VOID
|
||
KiSyncCacheTarget(
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Parameter1,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
);
|
||
|
||
ULONG_PTR
|
||
KiSyncMC_DrainTarget(
|
||
);
|
||
|
||
|
||
ULONG_PTR
|
||
KiSyncMC_Drain(
|
||
IN BOOLEAN AllProcessors,
|
||
IN PVOID BaseAddress,
|
||
IN ULONG Length
|
||
);
|
||
|
||
ULONG_PTR
|
||
KiSyncPrefetchVisibleTarget(
|
||
);
|
||
|
||
ULONG_PTR
|
||
KiSyncPrefetchVisible (
|
||
IN BOOLEAN AllProcessors,
|
||
IN PVOID BaseAddress,
|
||
IN ULONG Length
|
||
);
|
||
|
||
|
||
|
||
|
||
VOID
|
||
KiSyncCacheTarget (
|
||
IN PKIPI_CONTEXT SignalDone,
|
||
IN PVOID Parameter1,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
)
|
||
|
||
/*++
|
||
Routine Description:
|
||
|
||
This function synchronizes the I-fetch pipeline. Typically this routine will be
|
||
executed by every processor in the system in response to an IPI after the cache
|
||
is flushed. Each processor executing RFI while leaving the IPI produces the
|
||
serialization effect that is required after isync to make sure that further
|
||
instruction prefetches wait till the ISYNC completes.
|
||
|
||
Arguements:
|
||
|
||
SignalDone Supplies a pointer to a variable that is cleared when the
|
||
requested operation has been performed.
|
||
|
||
Parameter1 - Parameter3 - Not used.
|
||
|
||
Return Value:
|
||
|
||
Nothing.
|
||
--*/
|
||
{
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
__synci();
|
||
KiIpiSignalPacketDone(SignalDone);
|
||
|
||
#endif
|
||
return;
|
||
|
||
}
|
||
|
||
VOID
|
||
KeSweepIcache (
|
||
IN BOOLEAN AllProcessors
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function flushes the instruction cache on all processors that are
|
||
currently running threads which are children of the current process or
|
||
flushes the instruction cache on all processors in the host configuration.
|
||
|
||
N.B. Although PowerPC maintains cache coherency across processors, we
|
||
use the flash invalidate function (h/w) for I-Cache sweeps which doesn't
|
||
maintain coherency so we still do the MP I-Cache flush in s/w. plj.
|
||
|
||
Arguments:
|
||
|
||
AllProcessors - Supplies a boolean value that determines which instruction
|
||
caches are flushed.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
KIRQL OldIrql;
|
||
KAFFINITY TargetProcessors;
|
||
|
||
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
|
||
|
||
#if !defined(NT_UP)
|
||
//
|
||
// Acquire cache flush spinlock
|
||
// Cache flush is not MP safe yet
|
||
//
|
||
KeAcquireSpinLock(&KiCacheFlushLock, &OldIrql);
|
||
|
||
#endif
|
||
|
||
HalSweepIcache();
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
//
|
||
// Compute the set of target processors and send the sweep parameters
|
||
// to the target processors, if any, for execution.
|
||
//
|
||
|
||
TargetProcessors = KeActiveProcessors & PCR->NotMember;
|
||
if (TargetProcessors != 0) {
|
||
KiIpiSendPacket(TargetProcessors,
|
||
KiSweepIcacheTarget,
|
||
NULL,
|
||
NULL,
|
||
NULL);
|
||
}
|
||
|
||
|
||
//
|
||
// Wait until all target processors have finished sweeping their
|
||
// instruction caches.
|
||
//
|
||
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiStallOnPacketTargets(TargetProcessors);
|
||
}
|
||
|
||
//
|
||
// Lower IRQL to its previous level and return.
|
||
//
|
||
|
||
KeReleaseSpinLock(&KiCacheFlushLock, OldIrql);
|
||
|
||
#endif
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
KiSweepIcacheTarget (
|
||
IN PULONG SignalDone,
|
||
IN PVOID Parameter1,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This is the target function for sweeping the instruction cache on
|
||
target processors.
|
||
|
||
Arguments:
|
||
|
||
SignalDone Supplies a pointer to a variable that is cleared when the
|
||
requested operation has been performed.
|
||
|
||
Parameter1 - Parameter3 - Not used.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
//
|
||
// Sweep the instruction cache on the current processor and clear
|
||
// the sweep instruction cache packet address to signal the source
|
||
// to continue.
|
||
//
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
HalSweepIcache();
|
||
|
||
KiIpiSignalPacketDone(SignalDone);
|
||
|
||
#endif
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
KeSweepDcache (
|
||
IN BOOLEAN AllProcessors
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function flushes the data cache on all processors that are currently
|
||
running threads which are children of the current process or flushes the
|
||
data cache on all processors in the host configuration.
|
||
|
||
N.B. PowerPC maintains cache coherency across processors however
|
||
in this routine, the range of addresses being flushed is unknown
|
||
so we must still broadcast the request to the other processors.
|
||
|
||
Arguments:
|
||
|
||
AllProcessors - Supplies a boolean value that determines which data
|
||
caches are flushed.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
KIRQL OldIrql;
|
||
KAFFINITY TargetProcessors;
|
||
|
||
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
|
||
|
||
#if !defined(NT_UP)
|
||
//
|
||
// Acquire cache flush spinlock
|
||
// Cache flush is not MP safe yet
|
||
//
|
||
KeAcquireSpinLock(&KiCacheFlushLock, &OldIrql);
|
||
|
||
#endif
|
||
|
||
HalSweepDcache();
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
//
|
||
// Compute the set of target processors and send the sweep parameters
|
||
// to the target processors, if any, for execution.
|
||
//
|
||
|
||
TargetProcessors = KeActiveProcessors & PCR->NotMember;
|
||
if (TargetProcessors != 0) {
|
||
KiIpiSendPacket(TargetProcessors,
|
||
KiSweepDcacheTarget,
|
||
NULL,
|
||
NULL,
|
||
NULL);
|
||
}
|
||
|
||
|
||
//
|
||
// Wait until all target processors have finished sweeping their
|
||
// data caches.
|
||
//
|
||
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiStallOnPacketTargets(TargetProcessors);
|
||
}
|
||
|
||
//
|
||
// Lower IRQL to its previous level and return.
|
||
//
|
||
|
||
KeReleaseSpinLock(&KiCacheFlushLock, OldIrql);
|
||
|
||
#endif
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
KiSweepDcacheTarget (
|
||
IN PULONG SignalDone,
|
||
IN PVOID Parameter1,
|
||
IN PVOID Parameter2,
|
||
IN PVOID Parameter3
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This is the target function for sweeping the data cache on target
|
||
processors.
|
||
|
||
Arguments:
|
||
|
||
SignalDone Supplies a pointer to a variable that is cleared when the
|
||
requested operation has been performed.
|
||
|
||
Parameter1 - Parameter3 - Not used.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
//
|
||
// Sweep the data cache on the current processor and clear the sweep
|
||
// data cache packet address to signal the source to continue.
|
||
//
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
HalSweepDcache();
|
||
KiIpiSignalPacketDone(SignalDone);
|
||
|
||
#endif
|
||
|
||
return;
|
||
}
|
||
|
||
|
||
|
||
ULONG_PTR
|
||
KiSyncMC_DrainTarget(
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This is the target function for issuing PAL_MC_DRAIN to drain
|
||
prefetches, demand references and pending fc cache line evictions on the
|
||
target CPU it executes.
|
||
|
||
Argument:
|
||
|
||
None
|
||
|
||
|
||
Return Value:
|
||
|
||
Returns the status from the function HalCallPal
|
||
|
||
--*/
|
||
|
||
{
|
||
ULONG_PTR Status;
|
||
|
||
//
|
||
// Call HalCallPal to drain.
|
||
//
|
||
|
||
Status = HalCallPal(PAL_MC_DRAIN,
|
||
0,
|
||
0,
|
||
0,
|
||
0,
|
||
0,
|
||
0,
|
||
0);
|
||
|
||
ASSERT(Status == PAL_STATUS_SUCCESS);
|
||
|
||
return Status;
|
||
|
||
}
|
||
|
||
|
||
VOID
|
||
KeSweepCacheRange (
|
||
IN BOOLEAN AllProcessors,
|
||
IN PVOID BaseAddress,
|
||
IN ULONG Length
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function is used to flush a range of virtual addresses from both the
|
||
instruction and data cache on all processors in the system.
|
||
|
||
Irrespective of the length of the range, it should not call SweepIcache
|
||
or SweepDcache. This is because SweepDcache will only sweep D cache and
|
||
not the I cache and Vice versa. Since the caller of KeSweepCacheRange assumes
|
||
both the caches are being swept, one cannot call SweepIcache or SweepDcache
|
||
in trying to optimize.
|
||
|
||
|
||
Arguments:
|
||
|
||
AllProcessors - Not used
|
||
|
||
BaseAddress - Supplies a pointer to the base of the range that is flushed.
|
||
|
||
Length - Supplies the length of the range that is flushed if the base
|
||
address is specified.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
|
||
--*/
|
||
|
||
{
|
||
KIRQL OldIrql;
|
||
KAFFINITY TargetProcessors;
|
||
|
||
//
|
||
// We will not raise IRQL to synchronization level so that we can allow
|
||
// a context switch in between Flush Cache. FC need not run in the same processor
|
||
// throughout. It can be context switched. So no binding is done to any processor.
|
||
//
|
||
//
|
||
|
||
HalSweepCacheRange(BaseAddress,Length);
|
||
|
||
ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
|
||
|
||
//
|
||
// Raise IRQL to synchronization level to prevent a context switch.
|
||
//
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
OldIrql = KeRaiseIrqlToSynchLevel();
|
||
|
||
//
|
||
// Compute the set of target processors and send the sync parameters
|
||
// to the target processors, if any, for execution.
|
||
//
|
||
|
||
TargetProcessors = KeActiveProcessors & PCR->NotMember;
|
||
if (TargetProcessors != 0) {
|
||
KiIpiSendPacket(TargetProcessors,
|
||
KiSyncCacheTarget,
|
||
NULL,
|
||
NULL,
|
||
NULL);
|
||
}
|
||
|
||
#endif
|
||
|
||
//
|
||
// Synchronize the Instruction Prefetch pipe in the local processor.
|
||
//
|
||
|
||
__synci();
|
||
__isrlz();
|
||
|
||
//
|
||
// Wait until all target processors have finished sweeping the their
|
||
// data cache.
|
||
//
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiStallOnPacketTargets(TargetProcessors);
|
||
}
|
||
|
||
//
|
||
// Lower IRQL to its previous level and return.
|
||
//
|
||
|
||
KeLowerIrql(OldIrql);
|
||
|
||
#endif
|
||
|
||
return;
|
||
|
||
}
|
||
|
||
VOID
|
||
KeSweepIcacheRange (
|
||
IN BOOLEAN AllProcessors,
|
||
IN PVOID BaseAddress,
|
||
IN SIZE_T Length
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function is used to flush a range of virtual addresses from the
|
||
primary instruction cache on all processors in the host configuration.
|
||
|
||
If the length of the range is greater than the size of the
|
||
instruction cache, then one can call HalSweepIcache which calls
|
||
SAL to flush the entire cache. Since SAL does not take care of MP
|
||
flushing, HalSweepIcache has to use IPI mechanism to execute SAL
|
||
flush from each processor. We need to weight the overhead of all these
|
||
versus using HalSweepIcacheRange and avoiding IPI mechanism since
|
||
HalSweepIcacheRange uses fc instruction and fc instruction takes care of MP.
|
||
|
||
Arguments:
|
||
|
||
AllProcessors - Not used
|
||
|
||
BaseAddress - Supplies a pointer to the base of the range that is flushed.
|
||
|
||
Length - Supplies the length of the range that is flushed if the base
|
||
address is specified.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Note: For performance reason, we may update KeSweepIcacheRange to do the following:
|
||
if the range asked to sweep is very large, we may call KeSweepIcache to flush
|
||
the full cache.
|
||
|
||
|
||
|
||
--*/
|
||
|
||
{
|
||
KIRQL OldIrql;
|
||
KAFFINITY TargetProcessors;
|
||
|
||
//
|
||
// We will not raise IRQL to synchronization level so that we can allow
|
||
// a context switch in between Flush Cache. FC need not run in the same processor
|
||
// throughout. It can be context switched. So no binding is done to any processor.
|
||
//
|
||
//
|
||
|
||
HalSweepIcacheRange(BaseAddress,Length);
|
||
|
||
ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
|
||
|
||
//
|
||
// Raise IRQL to synchronization level to prevent a context switch.
|
||
//
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
OldIrql = KeRaiseIrqlToSynchLevel();
|
||
|
||
//
|
||
// Compute the set of target processors and send the sync parameters
|
||
// to the target processors, if any, for execution.
|
||
//
|
||
|
||
TargetProcessors = KeActiveProcessors & PCR->NotMember;
|
||
if (TargetProcessors != 0) {
|
||
KiIpiSendPacket(TargetProcessors,
|
||
KiSyncCacheTarget,
|
||
NULL,
|
||
NULL,
|
||
NULL);
|
||
}
|
||
|
||
#endif
|
||
|
||
//
|
||
// Synchronize the Instruction Prefetch pipe in the local processor.
|
||
//
|
||
|
||
__synci();
|
||
__isrlz();
|
||
|
||
//
|
||
// Wait until all target processors have finished sweeping the their
|
||
// data cache.
|
||
//
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiStallOnPacketTargets(TargetProcessors);
|
||
}
|
||
|
||
//
|
||
// Lower IRQL to its previous level and return.
|
||
//
|
||
|
||
KeLowerIrql(OldIrql);
|
||
|
||
#endif
|
||
|
||
return;
|
||
|
||
|
||
}
|
||
|
||
VOID
|
||
KeSweepCurrentIcacheRange (
|
||
IN PVOID BaseAddress,
|
||
IN SIZE_T Length
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function is used to flush a range of virtual addresses from the
|
||
primary instruction cache on the current processor.
|
||
|
||
This is used by the kernel debugger for flushing the i-cache after
|
||
modifying memory in case the instruction stream is changed.
|
||
|
||
To avoid calling SAL during phase 0 we use "fc" instead of the SAL cache
|
||
flush call.
|
||
|
||
Arguments:
|
||
|
||
BaseAddress - Supplies a pointer to the base of the range that is flushed.
|
||
|
||
Length - Supplies the length of the range that is flushed if the base
|
||
address is specified.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
KIRQL OldIrql;
|
||
|
||
KeRaiseIrql(HIGH_LEVEL, &OldIrql);
|
||
|
||
HalSweepIcacheRange(BaseAddress,Length);
|
||
|
||
//
|
||
// Synchronize the Instruction Prefetch pipe in the local processor.
|
||
//
|
||
|
||
__synci();
|
||
__isrlz();
|
||
|
||
KeLowerIrql(OldIrql);
|
||
|
||
return;
|
||
}
|
||
|
||
|
||
|
||
VOID
|
||
KeSweepDcacheRange (
|
||
IN BOOLEAN AllProcessors,
|
||
IN PVOID BaseAddress,
|
||
IN ULONG Length
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function is used to flush a range of virtual addresses from the
|
||
primary data cache on all processors in the host configuration.
|
||
|
||
If the length of the range is greater than the size of the
|
||
data cache, then one can call HalSweepDcache which calls
|
||
SAL to flush the entire cache. Since SAL does not take care of MP
|
||
flushing, HalSweepDcache has to use IPI mechanism to execute SAL
|
||
flush from each processor. We need to weight the overhead of all these
|
||
versus using HalSweepDcacheRange and avoiding IPI mechanism since
|
||
HalSweepDcacheRange uses fc instruction and fc instruction takes care of MP.
|
||
|
||
Arguments:
|
||
|
||
AllProcessors - Not used
|
||
|
||
BaseAddress - Supplies a pointer to the base of the range that is flushed.
|
||
|
||
Length - Supplies the length of the range that is flushed if the base
|
||
address is specified.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Note: For performance reason, we may update KeSweepDcacheRange to do the following:
|
||
if the range asked to sweep is very large, we may call KeSweepDcache to flush
|
||
the full cache.
|
||
|
||
|
||
|
||
--*/
|
||
|
||
{
|
||
KIRQL OldIrql;
|
||
KAFFINITY TargetProcessors;
|
||
|
||
//
|
||
// We will not raise IRQL to synchronization level so that we can allow
|
||
// a context switch in between Flush Cache. FC need not run in the same processor
|
||
// throughout. It can be context switched. So no binding is done to any processor.
|
||
//
|
||
//
|
||
|
||
HalSweepDcacheRange(BaseAddress,Length);
|
||
|
||
ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
|
||
|
||
//
|
||
// Raise IRQL to synchronization level to prevent a context switch.
|
||
//
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
OldIrql = KeRaiseIrqlToSynchLevel();
|
||
|
||
//
|
||
// Compute the set of target processors and send the sync parameters
|
||
// to the target processors, if any, for execution.
|
||
//
|
||
|
||
TargetProcessors = KeActiveProcessors & PCR->NotMember;
|
||
if (TargetProcessors != 0) {
|
||
KiIpiSendPacket(TargetProcessors,
|
||
KiSyncCacheTarget,
|
||
NULL,
|
||
NULL,
|
||
NULL);
|
||
}
|
||
|
||
#endif
|
||
|
||
//
|
||
// Synchronize the Instruction Prefetch pipe in the local processor.
|
||
//
|
||
|
||
__synci();
|
||
__isrlz();
|
||
|
||
//
|
||
// Wait until all target processors have finished sweeping the their
|
||
// data cache.
|
||
//
|
||
|
||
#if !defined(NT_UP)
|
||
|
||
if (TargetProcessors != 0) {
|
||
KiIpiStallOnPacketTargets(TargetProcessors);
|
||
}
|
||
|
||
//
|
||
// Lower IRQL to its previous level and return.
|
||
//
|
||
|
||
KeLowerIrql(OldIrql);
|
||
|
||
#endif
|
||
|
||
return;
|
||
|
||
|
||
}
|
||
|
||
ULONG_PTR
|
||
KiSyncMC_Drain (
|
||
IN BOOLEAN AllProcessors,
|
||
IN PVOID BaseAddress,
|
||
IN ULONG Length
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
KiSyncMC_Drain issues PAL_MC_DRAIN to drain either prefetches, demand references
|
||
or pending fc cache line evictions to all the processors in the system.
|
||
DrainTypePointer points to the variable, DrainType, which determines the type of
|
||
drain to be performed. This is typically used when changing the memory attribute
|
||
from WB to UC.
|
||
|
||
Arguments:
|
||
|
||
AllProcessors - All processors in the system.
|
||
|
||
BaseAddress - Supplies a pointer to the base of the range that is to be drained.
|
||
|
||
Length - Supplies the length of the range that is drained for the base
|
||
address specified.
|
||
|
||
Return Value:
|
||
|
||
Note: This is used when changing attributes of WB pages to UC pages.
|
||
|
||
|
||
--*/
|
||
|
||
{
|
||
ULONG_PTR Status;
|
||
//
|
||
// KiIpiGenericCall returns ULONG_PTR as the function value of the specified function
|
||
//
|
||
|
||
Status = (KiIpiGenericCall (
|
||
(PKIPI_BROADCAST_WORKER)KiSyncMC_DrainTarget,
|
||
(ULONG_PTR)NULL)
|
||
);
|
||
|
||
ASSERT(Status == PAL_STATUS_SUCCESS);
|
||
|
||
return Status;
|
||
|
||
|
||
}
|
||
|
||
ULONG_PTR
|
||
KiSyncPrefetchVisibleTarget(
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This is the target function for issuing PAL_PREFETCH VISIBILITY
|
||
on the target CPU it executes.
|
||
|
||
Argument:
|
||
|
||
Not used.
|
||
|
||
|
||
Return Value:
|
||
|
||
Returns the status from the function HalCallPal
|
||
|
||
--*/
|
||
|
||
{
|
||
ULONG_PTR Status;
|
||
|
||
//
|
||
// Call HalCallPal to drain.
|
||
//
|
||
|
||
Status = HalCallPal(PAL_PREFETCH_VISIBILITY,
|
||
0,
|
||
0,
|
||
0,
|
||
0,
|
||
0,
|
||
0,
|
||
0);
|
||
|
||
|
||
ASSERT(Status != PAL_STATUS_ERROR);
|
||
|
||
return Status;
|
||
|
||
}
|
||
|
||
|
||
|
||
ULONG_PTR
|
||
KiSyncPrefetchVisible (
|
||
IN BOOLEAN AllProcessors,
|
||
IN PVOID BaseAddress,
|
||
IN ULONG Length
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
KiSyncPrefetchVisible issues PAL_PREFETCH_VISIBILITY to cause the processor to make
|
||
all pending prefetches visible to subsequent fc instructions; or does nothing, on
|
||
processor implementations which does not require PAL support for disabling prefetch
|
||
in the architectural sequence. On processors that require PAL support for this
|
||
sequence, the actions performed by this procedure may include any or all
|
||
of the following (or none, as long as the processor guarantees that
|
||
prefetches that were issued prior to this call are not resident in the
|
||
processor's caches after the architected sequence is complete.
|
||
This is typically used when changing the memory attribute from WB to UC.
|
||
|
||
Arguments:
|
||
|
||
AllProcessors - All processors in the system.
|
||
|
||
BaseAddress - Supplies a pointer to the base of the range that is to be drained.
|
||
|
||
Length - Supplies the length of the range that is drained for the base
|
||
address specified.
|
||
|
||
Return Value:
|
||
|
||
Status of the PAL CALL
|
||
0 Success
|
||
1 Call not needed
|
||
-3 Error returned
|
||
|
||
Note: This is used when changing attributes of WB pages to UC pages.
|
||
|
||
|
||
--*/
|
||
|
||
{
|
||
ULONG_PTR Status;
|
||
|
||
switch (ProbePalVisibilitySupport) {
|
||
case 0:
|
||
if (NeedPalVisibilitySupport == 0)
|
||
Status = PAL_STATUS_SUPPORT_NOT_NEEDED;
|
||
else {
|
||
Status = (KiIpiGenericCall (
|
||
(PKIPI_BROADCAST_WORKER)KiSyncPrefetchVisibleTarget,
|
||
(ULONG_PTR)NULL)
|
||
);
|
||
}
|
||
break;
|
||
|
||
case 1:
|
||
Status = KiSyncPrefetchVisibleTarget();
|
||
|
||
ASSERT(Status != PAL_STATUS_ERROR);
|
||
|
||
ProbePalVisibilitySupport = 0;
|
||
|
||
if (Status == PAL_STATUS_SUPPORT_NOT_NEEDED) {
|
||
NeedPalVisibilitySupport = 0;
|
||
Status = PAL_STATUS_SUPPORT_NOT_NEEDED;
|
||
} else {
|
||
Status = (KiIpiGenericCall (
|
||
(PKIPI_BROADCAST_WORKER)KiSyncPrefetchVisibleTarget,
|
||
(ULONG_PTR)NULL)
|
||
);
|
||
}
|
||
break;
|
||
default:
|
||
Status = PAL_STATUS_ERROR;
|
||
break;
|
||
}
|
||
|
||
ASSERT(Status != PAL_STATUS_ERROR);
|
||
|
||
return Status;
|
||
|
||
}
|
||
|
||
|
||
|
||
VOID
|
||
KeSweepCacheRangeWithDrain (
|
||
IN BOOLEAN AllProcessors,
|
||
IN PVOID BaseAddress,
|
||
IN ULONG Length
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function is used to drain prefetches,demand references followed by flushing
|
||
the cache followed by draining pending fc cache line evictions to a specified range
|
||
address in all processors in the system.
|
||
|
||
|
||
Arguments:
|
||
|
||
AllProcessors - All processors in the system.
|
||
|
||
BaseAddress - Supplies a pointer to the base of the range that is flushed and drained.
|
||
|
||
Length - Supplies the length of the range that is flushed and drained for the base
|
||
address is specified.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Note: This is used when changing attributes of WB pages to UC pages.
|
||
|
||
--*/
|
||
|
||
{
|
||
ULONG_PTR Status;
|
||
|
||
Status = KiSyncPrefetchVisible(
|
||
AllProcessors,
|
||
BaseAddress,
|
||
Length
|
||
);
|
||
|
||
ASSERT(Status != PAL_STATUS_ERROR);
|
||
|
||
KeSweepCacheRange (
|
||
AllProcessors,
|
||
BaseAddress,
|
||
Length
|
||
);
|
||
|
||
Status = KiSyncMC_Drain (
|
||
AllProcessors,
|
||
BaseAddress,
|
||
Length
|
||
);
|
||
|
||
ASSERT(Status == PAL_STATUS_SUCCESS);
|
||
|
||
return;
|
||
|
||
|
||
}
|
||
|