able/windows-nt
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
windows-nt/Source/XPSP1/NT/base/ntos/ke/i386/mtrr.c

2049 lines
50 KiB
C
Raw Normal View History

Add source files
2020-09-26 03:20:57 -05:00
/*++
Copyright (c) 1991-2000 Microsoft Corporation
Module Name:
mtrr.c
Abstract:
This module implements interfaces that support manipulation of
memory type range registers.
These entry points only exist on x86 machines.
Author:
Ken Reneris (kenr) 11-Oct-95
Environment:
Kernel mode only.
Revision History:
--*/
#include "ki.h"
#include "mtrr.h"
#define STATIC
#define IDBG 0
#if DBG
#define DBGMSG(a) DbgPrint(a)
#else
#define DBGMSG(a)
#endif
//
// Internal declarations
//
//
// Range in generic terms
//
typedef struct _ONE_RANGE {
ULONGLONG Base;
ULONGLONG Limit;
UCHAR Type;
} ONE_RANGE, *PONE_RANGE;
#define GROW_RANGE_TABLE 4
//
// Range in specific mtrr terms
//
typedef struct _MTRR_RANGE {
MTRR_VARIABLE_BASE Base;
MTRR_VARIABLE_MASK Mask;
} MTRR_RANGE, *PMTRR_RANGE;
//
// System static information concerning cached range types
//
typedef struct _RANGE_INFO {
//
// Global MTRR info
//
MTRR_DEFAULT Default; // h/w mtrr default
MTRR_CAPABILITIES Capabilities; // h/w mtrr Capabilities
UCHAR DefaultCachedType; // default type for MmCached
//
// Variable MTRR information
//
BOOLEAN RangesValid; // Ranges initialized and valid.
BOOLEAN MtrrWorkaround; // Work Around needed/not.
UCHAR NoRange; // No ranges currently in Ranges
UCHAR MaxRange; // Max size of Ranges
PONE_RANGE Ranges; // Current ranges as set into h/w
} RANGE_INFO, *PRANGE_INFO;
//
// Structure used while processing range database
//
typedef struct _NEW_RANGE {
//
// Current Status
//
NTSTATUS Status;
//
// Generic info on new range
//
ULONGLONG Base;
ULONGLONG Limit;
UCHAR Type;
//
// MTRR image to be set into h/w
//
PMTRR_RANGE MTRR;
//
// RangeDatabase before edits were started
//
UCHAR NoRange;
PONE_RANGE Ranges;
//
// IPI context to coordinate concurrent processor update
//
ULONG NoMTRR;
PROCESSOR_LOCKSTEP Synchronize;
ULONG Processor;
} NEW_RANGE, *PNEW_RANGE;
//
// Prototypes
//
VOID
KiInitializeMTRR (
IN BOOLEAN LastProcessor
);
BOOLEAN
KiRemoveRange (
IN PNEW_RANGE NewRange,
IN ULONGLONG Base,
IN ULONGLONG Limit,
IN PBOOLEAN RemoveThisType
);
VOID
KiAddRange (
IN PNEW_RANGE NewRange,
IN ULONGLONG Base,
IN ULONGLONG Limit,
IN UCHAR Type
);
VOID
KiStartEffectiveRangeChange (
IN PNEW_RANGE NewRange
);
VOID
KiCompleteEffectiveRangeChange (
IN PNEW_RANGE NewRange
);
STATIC ULONG
KiRangeWeight (
IN PONE_RANGE Range
);
STATIC ULONG
KiFindFirstSetLeftBit (
IN ULONGLONG Set
);
STATIC ULONG
KiFindFirstSetRightBit (
IN ULONGLONG Set
);
VOID
KiLoadMTRRTarget (
IN PKIPI_CONTEXT SignalDone,
IN PVOID Context,
IN PVOID Parameter2,
IN PVOID Parameter3
);
NTSTATUS
KiLoadMTRR (
IN PNEW_RANGE Context
);
ULONGLONG
KiMaskToLength (
IN ULONGLONG Mask
);
ULONGLONG
KiLengthToMask (
IN ULONGLONG Length
);
#if IDBG
VOID
KiDumpMTRR (
PUCHAR DebugString,
PMTRR_RANGE MTRR
);
#endif
//
// --- AMD - Prototypes for AMD K6 MTRR Support functions. ---
//
NTSTATUS
KiAmdK6MtrrSetMemoryType (
IN ULONG BaseAddress,
IN ULONG NumberOfBytes,
IN MEMORY_CACHING_TYPE CacheType
);
VOID
KiAmdK6MtrrWRMSR (
VOID
);
// --- AMD - End ---
#ifdef ALLOC_PRAGMA
#pragma alloc_text(INIT,KiInitializeMTRR)
#pragma alloc_text(PAGELK,KiRemoveRange)
#pragma alloc_text(PAGELK,KiAddRange)
#pragma alloc_text(PAGELK,KiStartEffectiveRangeChange)
#pragma alloc_text(PAGELK,KiCompleteEffectiveRangeChange)
#pragma alloc_text(PAGELK,KiRangeWeight)
#pragma alloc_text(PAGELK,KiFindFirstSetLeftBit)
#pragma alloc_text(PAGELK,KiFindFirstSetRightBit)
#pragma alloc_text(PAGELK,KiLoadMTRR)
#pragma alloc_text(PAGELK,KiLoadMTRRTarget)
#pragma alloc_text(PAGELK,KiLockStepExecution)
#pragma alloc_text(PAGELK,KiLengthToMask)
#pragma alloc_text(PAGELK,KiMaskToLength)
#if IDBG
#pragma alloc_text(PAGELK,KiDumpMTRR)
#endif
#endif
//
// KiRangeLock - Used to synchronize accesses to KiRangeInfo
//
KSPIN_LOCK KiRangeLock;
//
// KiRangeInfo - Range type mapping information. Details specific h/w support
// and contains the current range database of how physical
// addresses have been set
RANGE_INFO KiRangeInfo;
VOID
KiInitializeMTRR (
IN BOOLEAN LastProcessor
)
/*++
Routine Description:
Called to incrementally initialize the physical range
database feature. First processor's MTRR set is read into the
physical range database.
Arguments:
LastProcessor - If set this is the last processor to execute this routine
such that when this processor finishes, the initialization is complete.
Return Value:
None - if there was a problem the function
KeSetPhysicalCacheTypeRange type is disabled.
--*/
{
BOOLEAN Status;
ULONG Index, Size;
MTRR_DEFAULT Default;
MTRR_CAPABILITIES Capabilities;
NEW_RANGE NewRange;
MTRR_VARIABLE_BASE MtrrBase;
MTRR_VARIABLE_MASK MtrrMask;
ULONGLONG Base, Mask, Length;
BOOLEAN RemoveThisType[MTRR_TYPE_MAX];
NTSTATUS NtStatus;
PKPRCB Prcb;
Status = TRUE;
RtlZeroMemory (&NewRange, sizeof (NewRange));
NewRange.Status = STATUS_UNSUCCESSFUL;
//
// If this is the first processor, initialize some fields
//
if (KeGetPcr()->Number == 0) {
KeInitializeSpinLock (&KiRangeLock);
KiRangeInfo.Capabilities.u.QuadPart = RDMSR(MTRR_MSR_CAPABILITIES);
KiRangeInfo.Default.u.QuadPart = RDMSR(MTRR_MSR_DEFAULT);
KiRangeInfo.DefaultCachedType = MTRR_TYPE_MAX;
//
// If h/w mtrr support is not enabled, disable OS support
//
if (!KiRangeInfo.Default.u.hw.MtrrEnabled ||
KiRangeInfo.Capabilities.u.hw.VarCnt == 0 ||
KiRangeInfo.Default.u.hw.Type != MTRR_TYPE_UC) {
DBGMSG("MTRR feature disabled.\n");
Status = FALSE;
} else {
//
// If USWC type is supported by hardware, but the MTRR
// feature is not set in KeFeatureBits, it is because
// the HAL indicated USWC should not be used on this
// machine. (Possibly due to shared memory clusters).
//
if (KiRangeInfo.Capabilities.u.hw.UswcSupported &&
((KeFeatureBits & KF_MTRR) == 0)) {
DBGMSG("KiInitializeMTRR: MTRR use globally disabled on this machine.\n");
KiRangeInfo.Capabilities.u.hw.UswcSupported = 0;
}
//
// Allocate initial range type database
//
KiRangeInfo.NoRange = 0;
KiRangeInfo.MaxRange = (UCHAR) KiRangeInfo.Capabilities.u.hw.VarCnt + GROW_RANGE_TABLE;
//
// Don't allocate a new range on reinitialization from
// hibernate.
//
if (KiRangeInfo.Ranges == NULL) {
KiRangeInfo.Ranges = ExAllocatePoolWithTag (NonPagedPool,
sizeof(ONE_RANGE) * KiRangeInfo.MaxRange,
' eK');
}
if (KiRangeInfo.Ranges != NULL) {
RtlZeroMemory (KiRangeInfo.Ranges,
sizeof(ONE_RANGE) * KiRangeInfo.MaxRange);
}
}
}
//
// Workaround for cpu signatures 611, 612, 616 and 617
// - if the request for setting a variable MTRR specifies
// an address which is not 4M aligned or length is not
// a multiple of 4M then possible problem for INVLPG inst.
// Detect if workaround is required
//
Prcb = KeGetCurrentPrcb();
if (Prcb->CpuType == 6 &&
(Prcb->CpuStep == 0x0101 || Prcb->CpuStep == 0x0102 ||
Prcb->CpuStep == 0x0106 || Prcb->CpuStep == 0x0107 )) {
if (strcmp(Prcb->VendorString, "GenuineIntel") == 0) {
//
// Only do this if it's an Intel part, other
// manufacturers may have the same stepping
// numbers but no bug.
//
KiRangeInfo.MtrrWorkaround = TRUE;
}
}
//
// If MTRR support disabled on first processor or if
// buffer not allocated then fall through
//
if (!KiRangeInfo.Ranges){
Status = FALSE;
} else {
//
// Verify MTRR support is symmetric
//
Capabilities.u.QuadPart = RDMSR(MTRR_MSR_CAPABILITIES);
if ((Capabilities.u.hw.UswcSupported) &&
((KeFeatureBits & KF_MTRR) == 0)) {
DBGMSG ("KiInitializeMTRR: setting UswcSupported FALSE\n");
Capabilities.u.hw.UswcSupported = 0;
}
Default.u.QuadPart = RDMSR(MTRR_MSR_DEFAULT);
if (Default.u.QuadPart != KiRangeInfo.Default.u.QuadPart ||
Capabilities.u.QuadPart != KiRangeInfo.Capabilities.u.QuadPart) {
DBGMSG ("KiInitializeMTRR: asymmetric mtrr support\n");
Status = FALSE;
}
}
NewRange.Status = STATUS_SUCCESS;
//
// MTRR registers should be identically set on each processor.
// Ranges should be added to the range database only for one
// processor.
//
if (Status && (KeGetPcr()->Number == 0)) {
#if IDBG
KiDumpMTRR ("Processor MTRR:", NULL);
#endif
//
// Read current MTRR settings for various cached range types
// and add them to the range database
//
for (Index=0; Index < Capabilities.u.hw.VarCnt; Index++) {
MtrrBase.u.QuadPart = RDMSR(MTRR_MSR_VARIABLE_BASE+Index*2);
MtrrMask.u.QuadPart = RDMSR(MTRR_MSR_VARIABLE_MASK+Index*2);
Mask = MtrrMask.u.QuadPart & MTRR_MASK_MASK;
Base = MtrrBase.u.QuadPart & MTRR_MASK_BASE;
//
// Note - the variable MTRR Mask does NOT contain the length
// spanned by the variable MTRR. Thus just checking the Valid
// Bit should be sufficient for identifying a valid MTRR.
//
if (MtrrMask.u.hw.Valid) {
Length = KiMaskToLength(Mask);
//
// Check for non-contiguous MTRR mask.
//
if ((Mask + Length) & MASK_OVERFLOW_MASK) {
DBGMSG ("KiInitializeMTRR: Found non-contiguous MTRR mask!\n");
Status = FALSE;
}
//
// Add this MTRR to the range database
//
Base &= Mask;
KiAddRange (
&NewRange,
Base,
Base + Length - 1,
(UCHAR) MtrrBase.u.hw.Type
);
//
// Check for default cache type
//
if (MtrrBase.u.hw.Type == MTRR_TYPE_WB) {
KiRangeInfo.DefaultCachedType = MTRR_TYPE_WB;
}
if (KiRangeInfo.DefaultCachedType == MTRR_TYPE_MAX &&
MtrrBase.u.hw.Type == MTRR_TYPE_WT) {
KiRangeInfo.DefaultCachedType = MTRR_TYPE_WT;
}
}
}
//
// If a default type for "cached" was not found, assume write-back
//
if (KiRangeInfo.DefaultCachedType == MTRR_TYPE_MAX) {
DBGMSG ("KiInitializeMTRR: assume write-back\n");
KiRangeInfo.DefaultCachedType = MTRR_TYPE_WB;
}
}
//
// Done
//
if (!NT_SUCCESS(NewRange.Status)) {
Status = FALSE;
}
if (!Status) {
DBGMSG ("KiInitializeMTRR: OS support for MTRRs disabled\n");
if (KiRangeInfo.Ranges != NULL) {
ExFreePool (KiRangeInfo.Ranges);
KiRangeInfo.Ranges = NULL;
}
} else {
// if last processor indicate initialization complete
if (LastProcessor) {
KiRangeInfo.RangesValid = TRUE;
}
}
}
VOID
KeRestoreMtrr (
VOID
)
/*++
Routine Description:
This function reloads the MTRR registers to be the current
known values. This is used on a system wakeup to ensure the
registers are sane.
N.B. The caller must have the PAGELK code locked
Arguments:
none
Return Value:
none
--*/
{
NEW_RANGE NewRange;
KIRQL OldIrql;
if (KiRangeInfo.RangesValid) {
RtlZeroMemory (&NewRange, sizeof (NewRange));
KeAcquireSpinLock (&KiRangeLock, &OldIrql);
KiStartEffectiveRangeChange (&NewRange);
ASSERT (NT_SUCCESS(NewRange.Status));
KiCompleteEffectiveRangeChange (&NewRange);
KeReleaseSpinLock (&KiRangeLock, OldIrql);
return;
}
//
// If the processor is a AMD K6 with MTRR support then perform
// processor specific implentaiton.
//
if (KeFeatureBits & KF_AMDK6MTRR) {
KeAcquireSpinLock (&KiRangeLock, &OldIrql);
KiLoadMTRR(NULL);
KeReleaseSpinLock (&KiRangeLock, OldIrql);
}
}
NTSTATUS
KeSetPhysicalCacheTypeRange (
IN PHYSICAL_ADDRESS PhysicalAddress,
IN ULONG NumberOfBytes,
IN MEMORY_CACHING_TYPE CacheType
)
/*++
Routine Description:
This function sets a physical range to a particular cache type.
If the system does not support setting cache policies based on
physical ranges, no action is taken.
Arguments:
PhysicalAddress - The starting address of the range being set
NumberOfBytes - The length, in bytes, of the range being set
CacheType - The caching type for which the physical range is
to be set to.
NonCached:
Setting ranges to be NonCached is done for
book keeping reasons. A return of SUCCESS when
setting a range NonCached does not mean it has
been physically set to as NonCached. The caller
must use a cache-disabled virtual pointer for
any NonCached range.
Cached:
A successful return indicates that the physical
range has been set to cached. This mode requires
the caller to be at irql < dispatch_level.
FrameBuffer:
A successful return indicates that the physical
range has been set to be framebuffer cached.
This mode requires the caller to be at irql <
dispatch_level.
USWCCached:
This type is to be satisfied only via PAT and
fails for the MTRR interface.
Return Value:
STATUS_SUCCESS - if success, the cache attributes of the physical range
have been set.
STATUS_NOT_SUPPORTED - either feature not supported or not yet initialized,
or MmWriteCombined type not supported and is
requested, or input range does not match restrictions
imposed by workarounds for current processor stepping
or is below 1M (in the fixed MTRR range), or not yet
initialized.
STATUS_UNSUCCESSFUL - Unable to satisfy request due to
- Unable to map software image into limited # of
hardware MTRRs.
- irql was not < DISPATCH_LEVEL.
- Failure due to other internal error (out of memory).
STATUS_INVALID_PARAMETER - Incorrect input memory type.
--*/
{
KIRQL OldIrql;
NEW_RANGE NewRange;
BOOLEAN RemoveThisType[MTRR_TYPE_MAX];
BOOLEAN EffectRangeChange, AddToRangeDatabase;
//
// If caller has requested the MmUSWCCached memory type then fail
// - MmUSWCCached is supported via PAT and not otherwise
//
if (CacheType == MmUSWCCached) {
return STATUS_NOT_SUPPORTED;
}
//
// Addresses above 4GB, below 1MB or not page aligned and
// page length are not supported.
//
if ((PhysicalAddress.HighPart != 0) ||
(PhysicalAddress.LowPart < (1 * 1024 * 1024)) ||
(PhysicalAddress.LowPart & 0xfff) ||
(NumberOfBytes & 0xfff) ) {
return STATUS_NOT_SUPPORTED;
}
ASSERT (NumberOfBytes != 0);
//
// If the processor is a AMD K6 with MTRR support then perform
// processor specific implentaiton.
//
if (KeFeatureBits & KF_AMDK6MTRR) {
if ((CacheType != MmWriteCombined) && (CacheType != MmNonCached)) {
return STATUS_NOT_SUPPORTED;
}
return KiAmdK6MtrrSetMemoryType(PhysicalAddress.LowPart,
NumberOfBytes,
CacheType);
}
//
// If processor doesn't have the memory type range feature
// return not supported.
//
if (!KiRangeInfo.RangesValid) {
return STATUS_NOT_SUPPORTED;
}
//
// Workaround for cpu signatures 611, 612, 616 and 617
// - if the request for setting a variable MTRR specifies
// an address which is not 4M aligned or length is not
// a multiple of 4M then return status not supported
//
if ((KiRangeInfo.MtrrWorkaround) &&
((PhysicalAddress.LowPart & 0x3fffff) ||
(NumberOfBytes & 0x3fffff))) {
return STATUS_NOT_SUPPORTED;
}
RtlZeroMemory (&NewRange, sizeof (NewRange));
NewRange.Base = PhysicalAddress.QuadPart;
NewRange.Limit = NewRange.Base + NumberOfBytes - 1;
//
// Determine what the new mtrr range type is. If setting NonCached then
// the database need not be updated to reflect the virtual change. This
// is because non-cached virtual pointers are mapped as cache disabled.
//
EffectRangeChange = TRUE;
AddToRangeDatabase = TRUE;
switch (CacheType) {
case MmNonCached:
NewRange.Type = MTRR_TYPE_UC;
//
// NonCached ranges do not need to be reflected into the h/w state
// as all non-cached ranges are mapped with cache-disabled pointers.
// This also means that cache-disabled ranges do not need to
// be put into mtrrs, or held in the range, regardless of the default
// range type.
//
EffectRangeChange = FALSE;
AddToRangeDatabase = FALSE;
break;
case MmCached:
NewRange.Type = KiRangeInfo.DefaultCachedType;
break;
case MmWriteCombined:
NewRange.Type = MTRR_TYPE_USWC;
//
// If USWC type isn't supported, then request can not be honored
//
if (!KiRangeInfo.Capabilities.u.hw.UswcSupported) {
DBGMSG ("KeSetPhysicalCacheTypeRange: USWC not supported\n");
return STATUS_NOT_SUPPORTED;
}
break;
default:
DBGMSG ("KeSetPhysicalCacheTypeRange: no such cache type\n");
return STATUS_INVALID_PARAMETER;
break;
}
NewRange.Status = STATUS_SUCCESS;
//
// The default type is UC thus the range is still mapped using
// a Cache Disabled VirtualPointer and hence it need not be added.
//
//
// If h/w needs updated, lock down the code required to effect the change
//
if (EffectRangeChange) {
if (KeGetCurrentIrql() >= DISPATCH_LEVEL) {
//
// Code can not be locked down. Supplying a new range type requires
// that the caller calls at irql < dispatch_level.
//
DBGMSG ("KeSetPhysicalCacheTypeRange failed due to calling IRQL == DISPATCH_LEVEL\n");
return STATUS_UNSUCCESSFUL;
}
MmLockPagableSectionByHandle(ExPageLockHandle);
}
//
// Serialize the range type database
//
KeAcquireSpinLock (&KiRangeLock, &OldIrql);
//
// If h/w is going to need updated, then start an effective range change
//
if (EffectRangeChange) {
KiStartEffectiveRangeChange (&NewRange);
}
if (NT_SUCCESS (NewRange.Status)) {
//
// If the new range is NonCached, then don't remove standard memory
// caching types
//
memset (RemoveThisType, TRUE, MTRR_TYPE_MAX);
if (NewRange.Type != MTRR_TYPE_UC) {
//
// If the requested type is uncached then the physical
// memory region is mapped using a cache disabled virtual pointer.
// The effective memory type for that region will be the lowest
// common denominator of the MTRR type and the cache type in the
// PTE. Therefore for a request of type UC, the effective type
// will be UC irrespective of the MTRR settings in that range.
// Hence it is not necessary to remove the existing MTRR settings
// (if any) for that range.
//
//
// Clip/remove any ranges in the target area
//
KiRemoveRange (&NewRange, NewRange.Base, NewRange.Limit, RemoveThisType);
}
//
// If needed, add new range type
//
if (AddToRangeDatabase) {
ASSERT (EffectRangeChange == TRUE);
KiAddRange (&NewRange, NewRange.Base, NewRange.Limit, NewRange.Type);
}
//
// If this is an effect range change, then complete it
//
if (EffectRangeChange) {
KiCompleteEffectiveRangeChange (&NewRange);
}
}
KeReleaseSpinLock (&KiRangeLock, OldIrql);
if (EffectRangeChange) {
MmUnlockPagableImageSection(ExPageLockHandle);
}
return NewRange.Status;
}
BOOLEAN
KiRemoveRange (
IN PNEW_RANGE NewRange,
IN ULONGLONG Base,
IN ULONGLONG Limit,
IN PBOOLEAN RemoveThisType
)
/*++
Routine Description:
This function removes any range overlapping with the passed range, of
type supplied in RemoveThisType from the global range database.
Arguments:
NewRange - Context information
Base - Base & Limit signify the first & last address of a range
Limit - which is to be removed from the range database
RemoveThisType - A TRUE flag for each type which can not overlap the
target range
Return Value:
TRUE - if the range database was altered such that it may no longer
be sorted.
--*/
{
ULONG i;
PONE_RANGE Range;
BOOLEAN DatabaseNeedsSorted;
DatabaseNeedsSorted = FALSE;
//
// Check each range
//
for (i=0, Range=KiRangeInfo.Ranges; i < KiRangeInfo.NoRange; i++, Range++) {
//
// If this range type doesn't need to be altered, skip it
//
if (!RemoveThisType[Range->Type]) {
continue;
}
//
// Check range to see if it overlaps with range being removed
//
if (Range->Base < Base) {
if (Range->Limit >= Base && Range->Limit <= Limit) {
//
// Truncate range to not overlap with area being removed
//
Range->Limit = Base - 1;
}
if (Range->Limit > Limit) {
//
// Target area is contained totally within this area.
// Split into two ranges
//
//
// Add range at end
//
DatabaseNeedsSorted = TRUE;
KiAddRange (
NewRange,
Limit+1,
Range->Limit,
Range->Type
);
//
// Turn current range into range at beginning
//
Range->Limit = Base - 1;
}
} else {
// Range->Base >= Base
if (Range->Base <= Limit) {
if (Range->Limit <= Limit) {
//
// This range is totally within the target area. Remove it.
//
DatabaseNeedsSorted = TRUE;
KiRangeInfo.NoRange -= 1;
Range->Base = KiRangeInfo.Ranges[KiRangeInfo.NoRange].Base;
Range->Limit = KiRangeInfo.Ranges[KiRangeInfo.NoRange].Limit;
Range->Type = KiRangeInfo.Ranges[KiRangeInfo.NoRange].Type;
//
// recheck at current location
//
i -= 1;
Range -= 1;
} else {
//
// Bump beginning past area being removed
//
Range->Base = Limit + 1;
}
}
}
}
if (!NT_SUCCESS (NewRange->Status)) {
DBGMSG ("KiRemoveRange: failure\n");
}
return DatabaseNeedsSorted;
}
VOID
KiAddRange (
IN PNEW_RANGE NewRange,
IN ULONGLONG Base,
IN ULONGLONG Limit,
IN UCHAR Type
)
/*++
Routine Description:
This function adds the passed range to the global range database.
Arguments:
NewRange - Context information
Base - Base & Limit signify the first & last address of a range
Limit - which is to be added to the range database
Type - Type of caching required for this range
Return Value:
None - Context is updated with an error if the table has overflowed
--*/
{
PONE_RANGE Range, OldRange;
ULONG size;
if (KiRangeInfo.NoRange >= KiRangeInfo.MaxRange) {
//
// Table is out of space, get a bigger one
//
OldRange = KiRangeInfo.Ranges;
size = sizeof(ONE_RANGE) * (KiRangeInfo.MaxRange + GROW_RANGE_TABLE);
Range = ExAllocatePoolWithTag (NonPagedPool, size, ' eK');
if (!Range) {
NewRange->Status = STATUS_UNSUCCESSFUL;
return ;
}
//
// Grow table
//
RtlZeroMemory (Range, size);
RtlCopyMemory (Range, OldRange, sizeof(ONE_RANGE) * KiRangeInfo.MaxRange);
KiRangeInfo.Ranges = Range;
KiRangeInfo.MaxRange += GROW_RANGE_TABLE;
ExFreePool (OldRange);
}
//
// Add new entry to table
//
KiRangeInfo.Ranges[KiRangeInfo.NoRange].Base = Base;
KiRangeInfo.Ranges[KiRangeInfo.NoRange].Limit = Limit;
KiRangeInfo.Ranges[KiRangeInfo.NoRange].Type = Type;
KiRangeInfo.NoRange += 1;
}
VOID
KiStartEffectiveRangeChange (
IN PNEW_RANGE NewRange
)
/*++
Routine Description:
This functions sets up the context information required to
track & later effect a range change in hardware
Arguments:
NewRange - Context information
Return Value:
None
--*/
{
ULONG size;
//
// Allocate working space for MTRR image
//
size = sizeof(MTRR_RANGE) * ((ULONG) KiRangeInfo.Capabilities.u.hw.VarCnt + 1);
NewRange->MTRR = ExAllocatePoolWithTag (NonPagedPool, size, ' eK');
if (!NewRange->MTRR) {
NewRange->Status = STATUS_UNSUCCESSFUL;
return ;
}
RtlZeroMemory (NewRange->MTRR, size);
//
// Save current range information in case of an error
//
size = sizeof(ONE_RANGE) * KiRangeInfo.NoRange;
NewRange->NoRange = KiRangeInfo.NoRange;
NewRange->Ranges = ExAllocatePoolWithTag (NonPagedPool, size, ' eK');
if (!NewRange->Ranges) {
NewRange->Status = STATUS_UNSUCCESSFUL;
return ;
}
RtlCopyMemory (NewRange->Ranges, KiRangeInfo.Ranges, size);
}
VOID
KiCompleteEffectiveRangeChange (
IN PNEW_RANGE NewRange
)
/*++
Routine Description:
This functions commits the range database to hardware, or backs
out the current changes to it.
Arguments:
NewRange - Context information
Return Value:
None
--*/
{
BOOLEAN Restart;
ULONG Index, Index2, RemIndex2, NoMTRR;
ULONGLONG BestLength, WhichMtrr;
ULONGLONG CurrLength;
ULONGLONG l, Base, Length, MLength;
PONE_RANGE Range;
ONE_RANGE OneRange;
PMTRR_RANGE MTRR;
BOOLEAN RoundDown;
BOOLEAN RemoveThisType[MTRR_TYPE_MAX];
PKPRCB Prcb;
KIRQL OldIrql, OldIrql2;
KAFFINITY TargetProcessors;
ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL);
Prcb = KeGetCurrentPrcb();
//
// Round all ranges, according to type, to match what h/w can support
//
for (Index=0; Index < KiRangeInfo.NoRange; Index++) {
Range = &KiRangeInfo.Ranges[Index];
//
// Determine rounding for this range type
//
RoundDown = TRUE;
if (Range->Type == MTRR_TYPE_UC) {
RoundDown = FALSE;
}
//
// Apply rounding
//
if (RoundDown) {
Range->Base = (Range->Base + MTRR_PAGE_SIZE - 1) & MTRR_PAGE_MASK;
Range->Limit = ((Range->Limit+1) & MTRR_PAGE_MASK)-1;
} else {
Range->Base = (Range->Base & MTRR_PAGE_MASK);
Range->Limit = ((Range->Limit + MTRR_PAGE_SIZE) & MTRR_PAGE_MASK)-1;
}
}
do {
Restart = FALSE;
//
// Sort the ranges by base address
//
for (Index=0; Index < KiRangeInfo.NoRange; Index++) {
Range = &KiRangeInfo.Ranges[Index];
for (Index2=Index+1; Index2 < KiRangeInfo.NoRange; Index2++) {
if (KiRangeInfo.Ranges[Index2].Base < Range->Base) {
//
// Swap KiRangeInfo.Ranges[Index] with KiRangeInfo.Ranges[Index2]
//
OneRange = *Range;
*Range = KiRangeInfo.Ranges[Index2];
KiRangeInfo.Ranges[Index2] = OneRange;
}
}
}
//
// At this point the range database is sorted on
// base address. Scan range database combining adjacent and
// overlapping ranges of the same type
//
for (Index=0; Index < (ULONG) KiRangeInfo.NoRange-1; Index++) {
Range = &KiRangeInfo.Ranges[Index];
//
// Scan the range database. If ranges are adjacent/overlap and are of
// the same type, combine them.
//
for (Index2 = Index+1; Index2 < (ULONG) KiRangeInfo.NoRange; Index2++) {
l = Range[0].Limit + 1;
if (l < Range[0].Limit) {
l = Range[0].Limit;
}
if (l >= KiRangeInfo.Ranges[Index2].Base &&
Range[0].Type == KiRangeInfo.Ranges[Index2].Type) {
//
// Increase Range[0] limit to cover Range[Index2]
//
if (KiRangeInfo.Ranges[Index2].Limit > Range[0].Limit) {
Range[0].Limit = KiRangeInfo.Ranges[Index2].Limit;
}
//
// Remove KiRangeInfo.Ranges[Index2]
//
if (Index2 < (ULONG) KiRangeInfo.NoRange - 1 ) {
//
// Copy everything from Index2 till end
// of range list. # Entries to copy is
// (KiRangeInfo.NoRange -1) - (Index2+1) + 1
//
RtlCopyMemory(
&(KiRangeInfo.Ranges[Index2]),
&(KiRangeInfo.Ranges[Index2+1]),
sizeof(ONE_RANGE) * (KiRangeInfo.NoRange-Index2-1)
);
}
KiRangeInfo.NoRange -= 1;
//
// Recheck current location
//
Index2 -= 1;
}
}
}
//
// At this point the range database is sorted on base
// address and adjacent/overlapping ranges of the same
// type are combined. Check for overlapping ranges -
// If legal then allow else truncate the less "weighty" range
//
for (Index = 0; Index < (ULONG) KiRangeInfo.NoRange-1 && !Restart; Index++) {
Range = &KiRangeInfo.Ranges[Index];
l = Range[0].Limit + 1;
if (l < Range[0].Limit) {
l = Range[0].Limit;
}
//
// If ranges overlap and are not of same type, and if the
// overlap is not legal then carve them to the best cache type
// available.
//
for (Index2 = Index+1; Index2 < (ULONG) KiRangeInfo.NoRange && !Restart; Index2++) {
if (l > KiRangeInfo.Ranges[Index2].Base) {
if (Range[0].Type == MTRR_TYPE_UC ||
KiRangeInfo.Ranges[Index2].Type == MTRR_TYPE_UC) {
//
// Overlap of a UC type with a range of any other type is
// legal
//
} else if ((Range[0].Type == MTRR_TYPE_WT &&
KiRangeInfo.Ranges[Index2].Type == MTRR_TYPE_WB) ||
(Range[0].Type == MTRR_TYPE_WB &&
KiRangeInfo.Ranges[Index2].Type == MTRR_TYPE_WT) ) {
//
// Overlap of WT and WB range is legal. The overlap range will
// be WT.
//
} else {
//
// This is an illegal overlap and we need to carve the ranges
// to remove the overlap.
//
// Pick range which has the cache type which should be used for
// the overlapped area
//
if (KiRangeWeight(&Range[0]) > KiRangeWeight(&(KiRangeInfo.Ranges[Index2]))){
RemIndex2 = Index2;
} else {
RemIndex2 = Index;
}
//
// Remove ranges of type which do not belong in the overlapped area
//
RtlZeroMemory (RemoveThisType, MTRR_TYPE_MAX);
RemoveThisType[KiRangeInfo.Ranges[RemIndex2].Type] = TRUE;
//
// Remove just the overlapped portion of the range.
//
Restart = KiRemoveRange (
NewRange,
KiRangeInfo.Ranges[Index2].Base,
(Range[0].Limit < KiRangeInfo.Ranges[Index2].Limit ?
Range[0].Limit : KiRangeInfo.Ranges[Index2].Limit),
RemoveThisType
);
}
}
}
}
} while (Restart);
//
// The range database is now rounded to fit in the h/w and sorted.
// Attempt to build MTRR settings which exactly describe the ranges
//
MTRR = NewRange->MTRR;
NoMTRR = 0;
for (Index=0;NT_SUCCESS(NewRange->Status)&& Index<KiRangeInfo.NoRange;Index++) {
Range = &KiRangeInfo.Ranges[Index];
//
// Build MTRRs to fit this range
//
Base = Range->Base;
Length = Range->Limit - Base + 1;
while (Length) {
//
// Compute MTRR length for current range base & length
//
if (Base == 0) {
MLength = Length;
} else {
MLength = (ULONGLONG) 1 << KiFindFirstSetRightBit(Base);
}
if (MLength > Length) {
MLength = Length;
}
l = (ULONGLONG) 1 << KiFindFirstSetLeftBit (MLength);
if (MLength > l) {
MLength = l;
}
//
// Store it in the next MTRR
//
MTRR[NoMTRR].Base.u.QuadPart = Base;
MTRR[NoMTRR].Base.u.hw.Type = Range->Type;
MTRR[NoMTRR].Mask.u.QuadPart = KiLengthToMask(MLength);
MTRR[NoMTRR].Mask.u.hw.Valid = 1;
NoMTRR += 1;
//
// Adjust off amount of data covered by that last MTRR
//
Base += MLength;
Length -= MLength;
//
// If there are too many MTRRs, and currently setting a
// Non-USWC range try to remove a USWC MTRR.
// (ie, convert some MmWriteCombined to MmNonCached).
//
if (NoMTRR > (ULONG) KiRangeInfo.Capabilities.u.hw.VarCnt) {
if (Range->Type != MTRR_TYPE_USWC) {
//
// Find smallest USWC type and drop it
//
// This is okay only if the default type is UC.
// Default type should always be UC unless BIOS changes
// it. Still ASSERT!
//
ASSERT(KiRangeInfo.Default.u.hw.Type == MTRR_TYPE_UC);
BestLength = (ULONGLONG) 1 << (MTRR_MAX_RANGE_SHIFT + 1);
for (Index2=0; Index2 < KiRangeInfo.Capabilities.u.hw.VarCnt; Index2++) {
if (MTRR[Index2].Base.u.hw.Type == MTRR_TYPE_USWC) {
CurrLength = KiMaskToLength(MTRR[Index2].Mask.u.QuadPart &
MTRR_MASK_MASK);
if (CurrLength < BestLength) {
WhichMtrr = Index2;
BestLength = CurrLength;
}
}
}
if (BestLength == ((ULONGLONG) 1 << (MTRR_MAX_RANGE_SHIFT + 1))) {
//
// Range was not found which could be dropped. Abort process
//
NewRange->Status = STATUS_UNSUCCESSFUL;
Length = 0;
} else {
//
// Remove WhichMtrr
//
NoMTRR -= 1;
MTRR[WhichMtrr] = MTRR[NoMTRR];
}
} else {
NewRange->Status = STATUS_UNSUCCESSFUL;
Length =0;
}
}
}
}
//
// Done building new MTRRs
//
if (NT_SUCCESS(NewRange->Status)) {
//
// Update the MTRRs on all processors
//
#if IDBG
KiDumpMTRR ("Loading the following MTRR:", NewRange->MTRR);
#endif
NewRange->Synchronize.TargetCount = 0;
NewRange->Synchronize.TargetPhase = &Prcb->ReverseStall;
NewRange->Synchronize.Processor = Prcb->Number;
//
// Previously enabled MTRRs with index > NoMTRR
// which could conflict with existing setting should be disabled
// This is taken care of by setting NewRange->NoMTRR to total
// number of variable MTRRs.
//
NewRange->NoMTRR = (ULONG) KiRangeInfo.Capabilities.u.hw.VarCnt;
//
// Synchronize with other IPI functions which may stall
//
KiLockContextSwap(&OldIrql);
#if !defined(NT_UP)
//
// Collect all the (other) processors
//
TargetProcessors = KeActiveProcessors & ~Prcb->SetMember;
if (TargetProcessors != 0) {
KiIpiSendSynchronousPacket (
Prcb,
TargetProcessors,
KiLoadMTRRTarget,
(PVOID) NewRange,
NULL,
NULL
);
//
// Wait for all processors to be collected
//
KiIpiStallOnPacketTargets(TargetProcessors);
//
// All processors are now waiting. Raise to high level to
// ensure this processor doesn't enter the debugger due to
// some interrupt service routine.
//
KeRaiseIrql (HIGH_LEVEL, &OldIrql2);
//
// There's no reason for any debug events now, so signal
// the other processors that they can all disable interrupts
// and begin the MTRR update
//
Prcb->ReverseStall += 1;
}
#endif
//
// Update MTRRs
//
KiLoadMTRR (NewRange);
//
// Release ContextSwap lock
//
KiUnlockContextSwap(OldIrql);
#if IDBG
KiDumpMTRR ("Processor MTRR:", NewRange->MTRR);
#endif
} else {
//
// There was an error, put original range database back
//
DBGMSG ("KiCompleteEffectiveRangeChange: mtrr update did not occur\n");
if (NewRange->Ranges) {
KiRangeInfo.NoRange = NewRange->NoRange;
RtlCopyMemory (
KiRangeInfo.Ranges,
NewRange->Ranges,
sizeof (ONE_RANGE) * KiRangeInfo.NoRange
);
}
}
//
// Cleanup
//
ExFreePool (NewRange->Ranges);
ExFreePool (NewRange->MTRR);
}
STATIC ULONG
KiRangeWeight (
IN PONE_RANGE Range
)
/*++
Routine Description:
This functions returns a weighting of the passed in range's cache
type. When two or more regions collide within the same h/w region
the types are weighted and that cache type of the higher weight
is used for the collision area.
Arguments:
Range - Range to obtain weighting for
Return Value:
The weight of the particular cache type
--*/
{
ULONG Weight;
switch (Range->Type) {
case MTRR_TYPE_UC: Weight = 5; break;
case MTRR_TYPE_USWC: Weight = 4; break;
case MTRR_TYPE_WP: Weight = 3; break;
case MTRR_TYPE_WT: Weight = 2; break;
case MTRR_TYPE_WB: Weight = 1; break;
default: Weight = 0; break;
}
return Weight;
}
STATIC ULONGLONG
KiMaskToLength (
IN ULONGLONG Mask
)
/*++
Routine Description:
This function returns the length specified by a particular
mtrr variable register mask.
--*/
{
if (Mask == 0) {
// Zero Mask signifies a length of 2**36
return(((ULONGLONG) 1 << MTRR_MAX_RANGE_SHIFT));
} else {
return(((ULONGLONG) 1 << KiFindFirstSetRightBit(Mask)));
}
}
STATIC ULONGLONG
KiLengthToMask (
IN ULONGLONG Length
)
/*++
Routine Description:
This function constructs the mask corresponding to the input length
to be set in a variable MTRR register. The length is assumed to be
a multiple of 4K.
--*/
{
ULONGLONG FullMask = 0xffffff;
if (Length == ((ULONGLONG) 1 << MTRR_MAX_RANGE_SHIFT)) {
return(0);
} else {
return(((FullMask << KiFindFirstSetRightBit(Length)) &
MTRR_RESVBIT_MASK));
}
}
STATIC ULONG
KiFindFirstSetRightBit (
IN ULONGLONG Set
)
/*++
Routine Description:
This function returns a bit position of the least significant
bit set in the passed ULONGLONG parameter. Passed parameter
must be non-zero.
--*/
{
ULONG bitno;
ASSERT(Set != 0);
for (bitno=0; !(Set & 0xFF); bitno += 8, Set >>= 8) ;
return KiFindFirstSetRight[Set & 0xFF] + bitno;
}
STATIC ULONG
KiFindFirstSetLeftBit (
IN ULONGLONG Set
)
/*++
Routine Description:
This function returns a bit position of the most significant
bit set in the passed ULONGLONG parameter. Passed parameter
must be non-zero.
--*/
{
ULONG bitno;
ASSERT(Set != 0);
for (bitno=56;!(Set & 0xFF00000000000000); bitno -= 8, Set <<= 8) ;
return KiFindFirstSetLeft[Set >> 56] + bitno;
}
#if IDBG
VOID
KiDumpMTRR (
PUCHAR DebugString,
PMTRR_RANGE MTRR
)
/*++
Routine Description:
This function dumps the MTRR information to the debugger
--*/
{
static PUCHAR Type[] = {
// 0 1 2 3 4 5 6
"UC ", "USWC", "????", "????", "WT ", "WP ", "WB " };
MTRR_VARIABLE_BASE Base;
MTRR_VARIABLE_MASK Mask;
ULONG Index;
ULONG i;
PUCHAR p;
DbgPrint ("%s\n", DebugString);
for (Index=0; Index < (ULONG) KiRangeInfo.Capabilities.u.hw.VarCnt; Index++) {
if (MTRR) {
Base = MTRR[Index].Base;
Mask = MTRR[Index].Mask;
} else {
Base.u.QuadPart = RDMSR(MTRR_MSR_VARIABLE_BASE+2*Index);
Mask.u.QuadPart = RDMSR(MTRR_MSR_VARIABLE_MASK+2*Index);
}
DbgPrint (" %d. ", Index);
if (Mask.u.hw.Valid) {
p = "????";
if (Base.u.hw.Type < 7) {
p = Type[Base.u.hw.Type];
}
DbgPrint ("%s %08x:%08x %08x:%08x",
p,
(ULONG) (Base.u.QuadPart >> 32),
((ULONG) (Base.u.QuadPart & MTRR_MASK_BASE)),
(ULONG) (Mask.u.QuadPart >> 32),
((ULONG) (Mask.u.QuadPart & MTRR_MASK_MASK))
);
}
DbgPrint ("\n");
}
}
#endif
VOID
KiLoadMTRRTarget (
IN PKIPI_CONTEXT SignalDone,
IN PVOID NewRange,
IN PVOID Parameter2,
IN PVOID Parameter3
)
{
PNEW_RANGE Context;
Context = (PNEW_RANGE) NewRange;
//
// Wait for all processors to be ready
//
KiIpiSignalPacketDoneAndStall(SignalDone,
Context->Synchronize.TargetPhase);
//
// Update MTRRs
//
KiLoadMTRR (Context);
}
#define MOV_EAX_CR4 _emit { 0Fh, 20h, E0h }
#define MOV_CR4_EAX _emit { 0Fh, 22h, E0h }
NTSTATUS
KiLoadMTRR (
IN PNEW_RANGE Context
)
/*++
Routine Description:
This function loads the memory type range registers into all processors
Arguments:
Context - Context which include the MTRRs to load
Return Value:
All processors are set into the new state
--*/
{
MTRR_DEFAULT Default;
BOOLEAN Enable;
ULONG HldCr0, HldCr4;
ULONG Index;
//
// Disable interrupts
//
Enable = KeDisableInterrupts();
//
// Synchronize all processors
//
if (!(KeFeatureBits & KF_AMDK6MTRR)) {
KiLockStepExecution (&Context->Synchronize);
}
_asm {
;
; Get current CR0
;
mov eax, cr0
mov HldCr0, eax
;
; Disable caching & line fill
;
and eax, not CR0_NW
or eax, CR0_CD
mov cr0, eax
;
; Flush caches
;
;
; wbinvd
;
_emit 0Fh
_emit 09h
;
; Get current cr4
;
_emit 0Fh
_emit 20h
_emit 0E0h ; mov eax, cr4
mov HldCr4, eax
;
; Disable global page
;
and eax, not CR4_PGE
_emit 0Fh
_emit 22h
_emit 0E0h ; mov cr4, eax
;
; Flush TLB
;
mov eax, cr3
mov cr3, eax
}
if (KeFeatureBits & KF_AMDK6MTRR) {
//
// Write the MTRRs
//
KiAmdK6MtrrWRMSR();
} else {
//
// Disable MTRRs
//
Default.u.QuadPart = RDMSR(MTRR_MSR_DEFAULT);
Default.u.hw.MtrrEnabled = 0;
WRMSR (MTRR_MSR_DEFAULT, Default.u.QuadPart);
//
// Load new MTRRs
//
for (Index=0; Index < Context->NoMTRR; Index++) {
WRMSR (MTRR_MSR_VARIABLE_BASE+2*Index, Context->MTRR[Index].Base.u.QuadPart);
WRMSR (MTRR_MSR_VARIABLE_MASK+2*Index, Context->MTRR[Index].Mask.u.QuadPart);
}
}
_asm {
;
; Flush caches (this should be a "nop", but it was in the Intel reference algorithm)
; This is required because of aggressive prefetch of both instr + data
;
;
; wbinvd
;
_emit 0Fh
_emit 09h
;
; Flush TLBs (same comment as above)
; Same explanation as above
;
mov eax, cr3
mov cr3, eax
}
if (!(KeFeatureBits & KF_AMDK6MTRR)) {
//
// Enable MTRRs
//
Default.u.hw.MtrrEnabled = 1;
WRMSR (MTRR_MSR_DEFAULT, Default.u.QuadPart);
}
_asm {
;
; Restore CR4 (global page enable)
;
mov eax, HldCr4
_emit 0Fh
_emit 22h
_emit 0E0h ; mov cr4, eax
;
; Restore CR0 (cache enable)
;
mov eax, HldCr0
mov cr0, eax
}
//
// Wait for all processors to reach the same place,
// restore interrupts and return.
//
if (!(KeFeatureBits & KF_AMDK6MTRR)) {
KiLockStepExecution (&Context->Synchronize);
}
KeEnableInterrupts (Enable);
return STATUS_SUCCESS;
}
VOID
KiLockStepExecution (
IN PPROCESSOR_LOCKSTEP Context
)
{
#if !defined(NT_UP)
ULONG CurrentPhase;
volatile ULONG *TargetPhase;
PKPRCB Prcb;
TargetPhase = Context->TargetPhase;
Prcb = KeGetCurrentPrcb();
if (Prcb->Number == (CCHAR) Context->Processor) {
//
// Wait for all processors to signal
//
while (Context->TargetCount != (ULONG) KeNumberProcessors - 1) {
KeYieldProcessor ();
}
//
// Reset count for next time
//
Context->TargetCount = 0;
//
// Let waiting processor go to next synchronization point
//
InterlockedIncrement ((PULONG) TargetPhase);
} else {
//
// Get current phase
//
CurrentPhase = *TargetPhase;
//
// Signal that we have completed the current phase
//
InterlockedIncrement ((PULONG) &Context->TargetCount);
//
// Wait for new phase to begin
//
while (*TargetPhase == CurrentPhase) {
KeYieldProcessor ();
}
}
#endif
}
Reference in a new issue Copy permalink