2759 lines
72 KiB
C
2759 lines
72 KiB
C
/*++
|
||
|
||
Copyright (c) 1989 Microsoft Corporation
|
||
|
||
Module Name:
|
||
|
||
pfnlist.c
|
||
|
||
Abstract:
|
||
|
||
This module contains the routines to manipulate pages within the
|
||
Page Frame Database.
|
||
|
||
Author:
|
||
|
||
Lou Perazzoli (loup) 4-Apr-1989
|
||
Landy Wang (landyw) 02-June-1997
|
||
|
||
Revision History:
|
||
|
||
--*/
|
||
#include "mi.h"
|
||
|
||
//
|
||
// The following line will generate an error is the number of colored
|
||
// free page lists is not 2, ie ZeroedPageList and FreePageList. If
|
||
// this number is changed, the size of the FreeCount array in the kernel
|
||
// node structure (KNODE) must be updated.
|
||
//
|
||
|
||
C_ASSERT(StandbyPageList == 2);
|
||
|
||
#define MM_LOW_LIMIT 2
|
||
|
||
KEVENT MmAvailablePagesEventHigh;
|
||
KEVENT MmAvailablePagesEventMedium;
|
||
|
||
PFN_NUMBER MmTransitionPrivatePages;
|
||
PFN_NUMBER MmTransitionSharedPages;
|
||
|
||
#define MI_TALLY_TRANSITION_PAGE_ADDITION(Pfn) \
|
||
if (Pfn->u3.e1.PrototypePte) { \
|
||
MmTransitionSharedPages += 1; \
|
||
} \
|
||
else { \
|
||
MmTransitionPrivatePages += 1; \
|
||
}
|
||
#if 0
|
||
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
|
||
#endif
|
||
|
||
#define MI_TALLY_TRANSITION_PAGE_REMOVAL(Pfn) \
|
||
if (Pfn->u3.e1.PrototypePte) { \
|
||
MmTransitionSharedPages -= 1; \
|
||
} \
|
||
else { \
|
||
MmTransitionPrivatePages -= 1; \
|
||
}
|
||
#if 0
|
||
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
|
||
#endif
|
||
|
||
//
|
||
// This counter is used to determine if standby pages are being cannibalized
|
||
// for use as free pages and therefore more aging should be attempted.
|
||
//
|
||
|
||
ULONG MiStandbyRemoved;
|
||
|
||
#if DBG
|
||
ULONG MiSaveStacks;
|
||
#endif
|
||
|
||
extern ULONG MmSystemShutdown;
|
||
|
||
PFN_NUMBER
|
||
FASTCALL
|
||
MiRemovePageByColor (
|
||
IN PFN_NUMBER Page,
|
||
IN ULONG PageColor
|
||
);
|
||
|
||
VOID
|
||
FASTCALL
|
||
MiLogPfnInformation (
|
||
IN PMMPFN Pfn1,
|
||
IN USHORT Reason
|
||
);
|
||
|
||
extern LOGICAL MiZeroingDisabled;
|
||
|
||
// #define _MI_DEBUG_PFN 1
|
||
|
||
#if defined (_MI_DEBUG_PFN)
|
||
|
||
#define MI_PFN_TRACE_MAX 0x8000
|
||
|
||
#define MI_PFN_BACKTRACE_LENGTH 6
|
||
|
||
typedef struct _MI_PFN_TRACES {
|
||
|
||
PFN_NUMBER PageFrameIndex;
|
||
MMLISTS Destination;
|
||
MMPTE PteContents;
|
||
PVOID Thread;
|
||
|
||
MMPFN Pfn;
|
||
|
||
PVOID StackTrace [MI_PFN_BACKTRACE_LENGTH];
|
||
|
||
} MI_PFN_TRACES, *PMI_PFN_TRACES;
|
||
|
||
LONG MiPfnIndex;
|
||
|
||
PMI_PFN_TRACES MiPfnTraces;
|
||
|
||
ULONG zpfn = 0x1;
|
||
|
||
VOID
|
||
FORCEINLINE
|
||
MiSnapPfn (
|
||
IN PMMPFN Pfn1,
|
||
IN MMLISTS Destination,
|
||
IN ULONG CallerId
|
||
)
|
||
{
|
||
MMPTE PteContents;
|
||
PMMPTE PointerPte;
|
||
PMI_PFN_TRACES Information;
|
||
ULONG Hash;
|
||
ULONG Index;
|
||
PEPROCESS Process;
|
||
|
||
if (MiPfnTraces == NULL) {
|
||
return;
|
||
}
|
||
|
||
if (zpfn & 0x1) {
|
||
|
||
if (Pfn1->PteAddress < MiGetPteAddress (MmPagedPoolStart)) {
|
||
return;
|
||
}
|
||
|
||
if (Pfn1->PteAddress > MiGetPteAddress (MmPagedPoolEnd)) {
|
||
return;
|
||
}
|
||
}
|
||
|
||
if (MmIsAddressValid (Pfn1->PteAddress)) {
|
||
PointerPte = Pfn1->PteAddress;
|
||
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte & ~0x1);
|
||
PteContents = *PointerPte;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// The containing page table page is not valid,
|
||
// map the page into hyperspace and reference it that way.
|
||
//
|
||
|
||
Process = PsGetCurrentProcess ();
|
||
PointerPte = MiMapPageInHyperSpaceAtDpc (Process, Pfn1->u4.PteFrame);
|
||
PointerPte = (PMMPTE)((PCHAR)PointerPte +
|
||
MiGetByteOffset(Pfn1->PteAddress));
|
||
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte & ~0x1);
|
||
PteContents = *PointerPte;
|
||
MiUnmapPageInHyperSpaceFromDpc (Process, PointerPte);
|
||
}
|
||
|
||
Index = InterlockedIncrement(&MiPfnIndex);
|
||
Index &= (MI_PFN_TRACE_MAX - 1);
|
||
Information = &MiPfnTraces[Index];
|
||
Information->PageFrameIndex = (Pfn1 - MmPfnDatabase);
|
||
Information->Destination = Destination;
|
||
Information->PteContents = PteContents;
|
||
Information->Thread = (PVOID)((ULONG_PTR)KeGetCurrentThread() | (CallerId));
|
||
Information->Pfn = *Pfn1;
|
||
RtlCaptureStackBackTrace (0, MI_PFN_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
|
||
}
|
||
|
||
#define MI_SNAP_PFN(_Pfn, dest, callerid) MiSnapPfn(_Pfn, dest, callerid)
|
||
|
||
#else
|
||
#define MI_SNAP_PFN(_Pfn, dest, callerid)
|
||
#endif
|
||
|
||
VOID
|
||
MiInitializePfnTracing (
|
||
VOID
|
||
)
|
||
{
|
||
#if defined (_MI_DEBUG_PFN)
|
||
MiPfnTraces = ExAllocatePoolWithTag (NonPagedPool,
|
||
MI_PFN_TRACE_MAX * sizeof (MI_PFN_TRACES),
|
||
'tCmM');
|
||
#endif
|
||
}
|
||
|
||
|
||
VOID
|
||
FASTCALL
|
||
MiInsertPageInFreeList (
|
||
IN PFN_NUMBER PageFrameIndex
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure inserts a page at the end of the free list.
|
||
|
||
Arguments:
|
||
|
||
PageFrameIndex - Supplies the physical page number to insert in the list.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Environment:
|
||
|
||
PFN lock held.
|
||
|
||
--*/
|
||
|
||
{
|
||
PFN_NUMBER last;
|
||
PMMPFN Pfn1;
|
||
PMMPFN Pfn2;
|
||
ULONG Color;
|
||
MMLISTS ListName;
|
||
PMMPFNLIST ListHead;
|
||
PMMCOLOR_TABLES ColorHead;
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
ASSERT ((PageFrameIndex != 0) &&
|
||
(PageFrameIndex <= MmHighestPhysicalPage) &&
|
||
(PageFrameIndex >= MmLowestPhysicalPage));
|
||
|
||
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
||
|
||
MI_SNAP_DATA (Pfn1, Pfn1->PteAddress, 8);
|
||
|
||
MI_SNAP_PFN(Pfn1, FreePageList, 0x1);
|
||
|
||
if (PERFINFO_IS_GROUP_ON(PERF_MEMORY)) {
|
||
MiLogPfnInformation (Pfn1, PERFINFO_LOG_TYPE_INSERTINFREELIST);
|
||
}
|
||
|
||
if (Pfn1->u3.e1.Rom == 1) {
|
||
|
||
//
|
||
// ROM pages do not go on free lists and are not counted towards
|
||
// MmAvailablePages as they are not reusable. Transform these pages
|
||
// into their pre-Phase 0 state and keep them off all lists.
|
||
//
|
||
|
||
ASSERT (XIPConfigured == TRUE);
|
||
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
||
ASSERT (Pfn1->u3.e1.Modified == 0);
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u4.InPageError == 0);
|
||
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
|
||
|
||
Pfn1->u1.Flink = 0;
|
||
Pfn1->u3.e1.PageLocation = 0;
|
||
|
||
return;
|
||
}
|
||
|
||
if (Pfn1->u3.e1.RemovalRequested == 1) {
|
||
Pfn1->u3.e1.CacheAttribute = MiNotMapped;
|
||
MiInsertPageInList (&MmBadPageListHead, PageFrameIndex);
|
||
return;
|
||
}
|
||
|
||
ListHead = &MmFreePageListHead;
|
||
|
||
ASSERT (Pfn1->u3.e1.LockCharged == 0);
|
||
|
||
PERFINFO_INSERTINLIST(PageFrameIndex, ListHead);
|
||
|
||
ListName = FreePageList;
|
||
|
||
#if DBG
|
||
#if defined (_X86_)
|
||
if ((MiSaveStacks != 0) && (MmFirstReservedMappingPte != NULL)) {
|
||
|
||
ULONG_PTR StackPointer;
|
||
ULONG_PTR StackBytes;
|
||
PULONG_PTR DataPage;
|
||
PEPROCESS Process;
|
||
|
||
_asm {
|
||
mov StackPointer, esp
|
||
}
|
||
|
||
MiZeroingDisabled = TRUE;
|
||
|
||
Process = PsGetCurrentProcess ();
|
||
|
||
DataPage = MiMapPageInHyperSpaceAtDpc (Process, PageFrameIndex);
|
||
|
||
DataPage[0] = StackPointer;
|
||
|
||
//
|
||
// For now, don't get fancy with copying more than what's in the current
|
||
// stack page. To do so would require checking the thread stack limits,
|
||
// DPC stack limits, etc.
|
||
//
|
||
|
||
StackBytes = PAGE_SIZE - BYTE_OFFSET(StackPointer);
|
||
DataPage[1] = StackBytes;
|
||
|
||
if (StackBytes != 0) {
|
||
|
||
if (StackBytes > MI_STACK_BYTES) {
|
||
StackBytes = MI_STACK_BYTES;
|
||
}
|
||
|
||
RtlCopyMemory ((PVOID)&DataPage[2], (PVOID)StackPointer, StackBytes);
|
||
}
|
||
|
||
MiUnmapPageInHyperSpaceFromDpc (Process, DataPage);
|
||
}
|
||
#endif
|
||
#endif
|
||
|
||
ASSERT (Pfn1->u4.VerifierAllocation == 0);
|
||
|
||
//
|
||
// Check to ensure the reference count for the page is zero.
|
||
//
|
||
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
|
||
ListHead->Total += 1; // One more page on the list.
|
||
|
||
last = ListHead->Blink;
|
||
|
||
if (last != MM_EMPTY_LIST) {
|
||
Pfn2 = MI_PFN_ELEMENT (last);
|
||
Pfn2->u1.Flink = PageFrameIndex;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// List is empty, add the page to the ListHead.
|
||
//
|
||
|
||
ListHead->Flink = PageFrameIndex;
|
||
}
|
||
|
||
ListHead->Blink = PageFrameIndex;
|
||
Pfn1->u1.Flink = MM_EMPTY_LIST;
|
||
Pfn1->u2.Blink = last;
|
||
|
||
Pfn1->u3.e1.PageLocation = ListName;
|
||
Pfn1->u3.e1.CacheAttribute = MiNotMapped;
|
||
Pfn1->u4.InPageError = 0;
|
||
|
||
//
|
||
// Update the count of usable pages in the system. If the count
|
||
// transitions from 0 to 1, the event associated with available
|
||
// pages should become true.
|
||
//
|
||
|
||
MmAvailablePages += 1;
|
||
|
||
//
|
||
// A page has just become available, check to see if the
|
||
// page wait events should be signaled.
|
||
//
|
||
|
||
if (MmAvailablePages <= MM_HIGH_LIMIT) {
|
||
if (MmAvailablePages == MM_HIGH_LIMIT) {
|
||
KeSetEvent (&MmAvailablePagesEventHigh, 0, FALSE);
|
||
}
|
||
else if (MmAvailablePages == MM_MEDIUM_LIMIT) {
|
||
KeSetEvent (&MmAvailablePagesEventMedium, 0, FALSE);
|
||
}
|
||
else if (MmAvailablePages == MM_LOW_LIMIT) {
|
||
KeSetEvent (&MmAvailablePagesEvent, 0, FALSE);
|
||
}
|
||
}
|
||
|
||
//
|
||
// Signal applications if the freed page crosses a threshold.
|
||
//
|
||
|
||
if (MmAvailablePages == MmLowMemoryThreshold) {
|
||
KeClearEvent (MiLowMemoryEvent);
|
||
}
|
||
else if (MmAvailablePages == MmHighMemoryThreshold) {
|
||
KeSetEvent (MiHighMemoryEvent, 0, FALSE);
|
||
}
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
//
|
||
// Increment the free page count for this node.
|
||
//
|
||
|
||
if (KeNumberNodes > 1) {
|
||
KeNodeBlock[Pfn1->u3.e1.PageColor]->FreeCount[ListName]++;
|
||
}
|
||
#endif
|
||
|
||
//
|
||
// We are adding a page to the free page list.
|
||
// Add the page to the end of the correct colored page list.
|
||
//
|
||
|
||
Color = MI_GET_COLOR_FROM_LIST_ENTRY(PageFrameIndex, Pfn1);
|
||
|
||
ColorHead = &MmFreePagesByColor[ListName][Color];
|
||
|
||
if (ColorHead->Flink == MM_EMPTY_LIST) {
|
||
|
||
//
|
||
// This list is empty, add this as the first and last
|
||
// entry.
|
||
//
|
||
|
||
ColorHead->Flink = PageFrameIndex;
|
||
ColorHead->Blink = Pfn1;
|
||
}
|
||
else {
|
||
Pfn2 = (PMMPFN)ColorHead->Blink;
|
||
Pfn2->OriginalPte.u.Long = PageFrameIndex;
|
||
ColorHead->Blink = Pfn1;
|
||
}
|
||
ColorHead->Count += 1;
|
||
Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
|
||
|
||
if ((ListHead->Total >= MmMinimumFreePagesToZero) &&
|
||
(MmZeroingPageThreadActive == FALSE)) {
|
||
|
||
//
|
||
// There are enough pages on the free list, start
|
||
// the zeroing page thread.
|
||
//
|
||
|
||
MmZeroingPageThreadActive = TRUE;
|
||
KeSetEvent (&MmZeroingPageEvent, 0, FALSE);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
FASTCALL
|
||
MiInsertPageInList (
|
||
IN PMMPFNLIST ListHead,
|
||
IN PFN_NUMBER PageFrameIndex
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure inserts a page at the end of the specified list (standby,
|
||
bad, zeroed, modified).
|
||
|
||
Arguments:
|
||
|
||
ListHead - Supplies the list of the list in which to insert the
|
||
specified physical page.
|
||
|
||
PageFrameIndex - Supplies the physical page number to insert in the list.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Environment:
|
||
|
||
Must be holding the PFN database lock with APCs disabled.
|
||
|
||
--*/
|
||
|
||
{
|
||
PFN_NUMBER last;
|
||
PMMPFN Pfn1;
|
||
PMMPFN Pfn2;
|
||
ULONG Color;
|
||
MMLISTS ListName;
|
||
#if MI_BARRIER_SUPPORTED
|
||
ULONG BarrierStamp;
|
||
#endif
|
||
|
||
ASSERT (ListHead != &MmFreePageListHead);
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
ASSERT ((PageFrameIndex != 0) &&
|
||
(PageFrameIndex <= MmHighestPhysicalPage) &&
|
||
(PageFrameIndex >= MmLowestPhysicalPage));
|
||
|
||
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
||
|
||
ASSERT (Pfn1->u3.e1.LockCharged == 0);
|
||
|
||
PERFINFO_INSERTINLIST(PageFrameIndex, ListHead);
|
||
|
||
ListName = ListHead->ListName;
|
||
|
||
MI_SNAP_DATA (Pfn1, Pfn1->PteAddress, 0x20 + ListName);
|
||
|
||
MI_SNAP_PFN(Pfn1, ListName, 0x2);
|
||
|
||
#if DBG
|
||
if (MmDebug & MM_DBG_PAGE_REF_COUNT) {
|
||
|
||
if ((ListName == StandbyPageList) || (ListName == ModifiedPageList)) {
|
||
|
||
PMMPTE PointerPte;
|
||
PEPROCESS Process;
|
||
|
||
if ((Pfn1->u3.e1.PrototypePte == 1) &&
|
||
(MmIsAddressValid (Pfn1->PteAddress))) {
|
||
Process = NULL;
|
||
PointerPte = Pfn1->PteAddress;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// The page containing the prototype PTE is not valid,
|
||
// map the page into hyperspace and reference it that way.
|
||
//
|
||
|
||
Process = PsGetCurrentProcess ();
|
||
PointerPte = MiMapPageInHyperSpaceAtDpc (Process, Pfn1->u4.PteFrame);
|
||
PointerPte = (PMMPTE)((PCHAR)PointerPte +
|
||
MiGetByteOffset(Pfn1->PteAddress));
|
||
}
|
||
|
||
ASSERT ((MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) == PageFrameIndex) ||
|
||
(MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) == PageFrameIndex));
|
||
ASSERT (PointerPte->u.Soft.Transition == 1);
|
||
ASSERT (PointerPte->u.Soft.Prototype == 0);
|
||
if (Process != NULL) {
|
||
MiUnmapPageInHyperSpaceFromDpc (Process, PointerPte);
|
||
}
|
||
}
|
||
}
|
||
|
||
if ((ListName == StandbyPageList) || (ListName == ModifiedPageList)) {
|
||
if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
|
||
(Pfn1->OriginalPte.u.Soft.Transition == 1)) {
|
||
KeBugCheckEx (MEMORY_MANAGEMENT, 0x8888, 0,0,0);
|
||
}
|
||
}
|
||
#endif
|
||
|
||
//
|
||
// Check to ensure the reference count for the page is zero.
|
||
//
|
||
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
|
||
if (Pfn1->u3.e1.Rom == 1) {
|
||
|
||
//
|
||
// ROM pages do not go on transition lists and are not counted towards
|
||
// MmAvailablePages as they are not reusable. Migrate these pages
|
||
// into a separate list but keep the PageLocation as standby.
|
||
//
|
||
|
||
ASSERT (XIPConfigured == TRUE);
|
||
ASSERT (Pfn1->u3.e1.Modified == 0);
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
||
|
||
ListHead = &MmRomPageListHead;
|
||
ListHead->Total += 1; // One more page on the list.
|
||
last = ListHead->Blink;
|
||
|
||
if (last != MM_EMPTY_LIST) {
|
||
Pfn2 = MI_PFN_ELEMENT (last);
|
||
Pfn2->u1.Flink = PageFrameIndex;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// List is empty, add the page to the ListHead.
|
||
//
|
||
|
||
ListHead->Flink = PageFrameIndex;
|
||
}
|
||
|
||
ListHead->Blink = PageFrameIndex;
|
||
Pfn1->u1.Flink = MM_EMPTY_LIST;
|
||
Pfn1->u2.Blink = last;
|
||
|
||
Pfn1->u3.e1.PageLocation = ListName;
|
||
|
||
return;
|
||
}
|
||
|
||
ListHead->Total += 1; // One more page on the list.
|
||
|
||
if (ListHead == &MmModifiedPageListHead) {
|
||
|
||
//
|
||
// Leave the page as cached as it may need to be written out at some
|
||
// point and presumably the driver stack will need to map it at that
|
||
// time.
|
||
//
|
||
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
||
|
||
//
|
||
// On MIPS R4000 modified pages destined for the paging file are
|
||
// kept on separate lists which group pages of the same color
|
||
// together.
|
||
//
|
||
|
||
if (Pfn1->OriginalPte.u.Soft.Prototype == 0) {
|
||
|
||
//
|
||
// This page is destined for the paging file (not
|
||
// a mapped file). Change the list head to the
|
||
// appropriate colored list head.
|
||
//
|
||
|
||
#if MM_MAXIMUM_NUMBER_OF_COLORS > 1
|
||
ListHead = &MmModifiedPageListByColor [Pfn1->u3.e1.PageColor];
|
||
#else
|
||
ListHead = &MmModifiedPageListByColor [0];
|
||
#endif
|
||
ASSERT (ListHead->ListName == ListName);
|
||
ListHead->Total += 1;
|
||
MmTotalPagesForPagingFile += 1;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// This page is destined for a mapped file (not
|
||
// the paging file). If there are no other pages currently
|
||
// destined for the mapped file, start our timer so that we can
|
||
// ensure that these pages make it to disk even if we don't pile
|
||
// up enough of them to trigger the modified page writer or need
|
||
// the memory. If we don't do this here, then for this scenario,
|
||
// only an orderly system shutdown will write them out (days,
|
||
// weeks, months or years later) and any power out in between
|
||
// means we'll have lost the data.
|
||
//
|
||
|
||
if (ListHead->Total - MmTotalPagesForPagingFile == 1) {
|
||
|
||
//
|
||
// Start the DPC timer because we're the first on the list.
|
||
//
|
||
|
||
if (MiTimerPending == FALSE) {
|
||
MiTimerPending = TRUE;
|
||
|
||
KeSetTimerEx (&MiModifiedPageWriterTimer,
|
||
MiModifiedPageLife,
|
||
0,
|
||
&MiModifiedPageWriterTimerDpc);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
else if ((Pfn1->u3.e1.RemovalRequested == 1) &&
|
||
(ListName <= StandbyPageList)) {
|
||
|
||
ListHead->Total -= 1; // Undo previous increment
|
||
|
||
if (ListName == StandbyPageList) {
|
||
Pfn1->u3.e1.PageLocation = StandbyPageList;
|
||
MiRestoreTransitionPte (PageFrameIndex);
|
||
}
|
||
|
||
Pfn1->u3.e1.CacheAttribute = MiNotMapped;
|
||
|
||
ListHead = &MmBadPageListHead;
|
||
ASSERT (ListHead->ListName == BadPageList);
|
||
ListHead->Total += 1; // One more page on the list.
|
||
ListName = BadPageList;
|
||
}
|
||
|
||
last = ListHead->Blink;
|
||
|
||
if (last != MM_EMPTY_LIST) {
|
||
Pfn2 = MI_PFN_ELEMENT (last);
|
||
Pfn2->u1.Flink = PageFrameIndex;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// List is empty, add the page to the ListHead.
|
||
//
|
||
|
||
ListHead->Flink = PageFrameIndex;
|
||
}
|
||
|
||
ListHead->Blink = PageFrameIndex;
|
||
Pfn1->u1.Flink = MM_EMPTY_LIST;
|
||
Pfn1->u2.Blink = last;
|
||
|
||
Pfn1->u3.e1.PageLocation = ListName;
|
||
|
||
//
|
||
// If the page was placed on the standby or zeroed list,
|
||
// update the count of usable pages in the system. If the count
|
||
// transitions from 0 to 1, the event associated with available
|
||
// pages should become true.
|
||
//
|
||
|
||
if (ListName <= StandbyPageList) {
|
||
|
||
MmAvailablePages += 1;
|
||
|
||
//
|
||
// A page has just become available, check to see if the
|
||
// page wait events should be signaled.
|
||
//
|
||
|
||
if (MmAvailablePages <= MM_HIGH_LIMIT) {
|
||
if (MmAvailablePages == MM_HIGH_LIMIT) {
|
||
KeSetEvent (&MmAvailablePagesEventHigh, 0, FALSE);
|
||
}
|
||
else if (MmAvailablePages == MM_MEDIUM_LIMIT) {
|
||
KeSetEvent (&MmAvailablePagesEventMedium, 0, FALSE);
|
||
}
|
||
else if (MmAvailablePages == MM_LOW_LIMIT) {
|
||
KeSetEvent (&MmAvailablePagesEvent, 0, FALSE);
|
||
}
|
||
}
|
||
|
||
//
|
||
// Signal applications if the freed page crosses a threshold.
|
||
//
|
||
|
||
if (MmAvailablePages == MmLowMemoryThreshold) {
|
||
KeClearEvent (MiLowMemoryEvent);
|
||
}
|
||
else if (MmAvailablePages == MmHighMemoryThreshold) {
|
||
KeSetEvent (MiHighMemoryEvent, 0, FALSE);
|
||
}
|
||
|
||
if (ListName <= FreePageList) {
|
||
|
||
PMMCOLOR_TABLES ColorHead;
|
||
|
||
ASSERT (ListName == ZeroedPageList);
|
||
ASSERT (Pfn1->u4.InPageError == 0);
|
||
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
//
|
||
// Increment the zero page count for this node.
|
||
//
|
||
|
||
if (KeNumberNodes > 1) {
|
||
KeNodeBlock[Pfn1->u3.e1.PageColor]->FreeCount[ListName]++;
|
||
}
|
||
#endif
|
||
|
||
//
|
||
// We are adding a page to the zeroed page list.
|
||
// Add the page to the end of the correct colored page list.
|
||
//
|
||
|
||
Color = MI_GET_COLOR_FROM_LIST_ENTRY(PageFrameIndex, Pfn1);
|
||
|
||
ColorHead = &MmFreePagesByColor[ListName][Color];
|
||
|
||
if (ColorHead->Flink == MM_EMPTY_LIST) {
|
||
|
||
//
|
||
// This list is empty, add this as the first and last
|
||
// entry.
|
||
//
|
||
|
||
ColorHead->Flink = PageFrameIndex;
|
||
ColorHead->Blink = (PVOID)Pfn1;
|
||
}
|
||
else {
|
||
Pfn2 = (PMMPFN)ColorHead->Blink;
|
||
Pfn2->OriginalPte.u.Long = PageFrameIndex;
|
||
ColorHead->Blink = (PVOID)Pfn1;
|
||
}
|
||
Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
|
||
|
||
ColorHead->Count += 1;
|
||
|
||
#if MI_BARRIER_SUPPORTED
|
||
if (ListName == ZeroedPageList) {
|
||
MI_BARRIER_STAMP_ZEROED_PAGE (&BarrierStamp);
|
||
Pfn1->u4.PteFrame = BarrierStamp;
|
||
}
|
||
#endif
|
||
}
|
||
else {
|
||
|
||
//
|
||
// Transition page list so tally it appropriately.
|
||
//
|
||
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
||
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
//
|
||
// Check to see if there are too many modified pages.
|
||
//
|
||
|
||
if (ListName == ModifiedPageList) {
|
||
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
||
|
||
//
|
||
// Transition page list so tally it appropriately.
|
||
//
|
||
|
||
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
||
|
||
if (Pfn1->OriginalPte.u.Soft.Prototype == 0) {
|
||
ASSERT (Pfn1->OriginalPte.u.Soft.PageFileHigh == 0);
|
||
}
|
||
|
||
PsGetCurrentProcess()->ModifiedPageCount += 1;
|
||
|
||
if ((MmModifiedPageListHead.Total >= MmModifiedPageMaximum) &&
|
||
(MmAvailablePages < MmMoreThanEnoughFreePages)) {
|
||
|
||
//
|
||
// Start the modified page writer.
|
||
//
|
||
|
||
KeSetEvent (&MmModifiedPageWriterEvent, 0, FALSE);
|
||
}
|
||
}
|
||
else if (ListName == ModifiedNoWritePageList) {
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
||
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
|
||
VOID
|
||
FASTCALL
|
||
MiInsertStandbyListAtFront (
|
||
IN PFN_NUMBER PageFrameIndex
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure inserts a page at the front of the standby list.
|
||
|
||
Arguments:
|
||
|
||
PageFrameIndex - Supplies the physical page number to insert in the list.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Environment:
|
||
|
||
PFN lock held.
|
||
|
||
--*/
|
||
|
||
{
|
||
PFN_NUMBER first;
|
||
PFN_NUMBER last;
|
||
IN PMMPFNLIST ListHead;
|
||
PMMPFN Pfn1;
|
||
PMMPFN Pfn2;
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
ASSERT ((PageFrameIndex != 0) && (PageFrameIndex <= MmHighestPhysicalPage) &&
|
||
(PageFrameIndex >= MmLowestPhysicalPage));
|
||
|
||
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
||
|
||
MI_SNAP_DATA (Pfn1, Pfn1->PteAddress, 9);
|
||
|
||
PERFINFO_INSERT_FRONT_STANDBY(PageFrameIndex);
|
||
|
||
#if DBG
|
||
if (MmDebug & MM_DBG_PAGE_REF_COUNT) {
|
||
|
||
PMMPTE PointerPte;
|
||
PEPROCESS Process;
|
||
|
||
if ((Pfn1->u3.e1.PrototypePte == 1) &&
|
||
(MmIsAddressValid (Pfn1->PteAddress))) {
|
||
PointerPte = Pfn1->PteAddress;
|
||
Process = NULL;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// The page containing the prototype PTE is not valid,
|
||
// map the page into hyperspace and reference it that way.
|
||
//
|
||
|
||
Process = PsGetCurrentProcess ();
|
||
PointerPte = MiMapPageInHyperSpaceAtDpc (Process, Pfn1->u4.PteFrame);
|
||
PointerPte = (PMMPTE)((PCHAR)PointerPte +
|
||
MiGetByteOffset(Pfn1->PteAddress));
|
||
}
|
||
|
||
ASSERT ((MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) == PageFrameIndex) ||
|
||
(MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) == PageFrameIndex));
|
||
ASSERT (PointerPte->u.Soft.Transition == 1);
|
||
ASSERT (PointerPte->u.Soft.Prototype == 0);
|
||
if (Process != NULL) {
|
||
MiUnmapPageInHyperSpaceFromDpc (Process, PointerPte);
|
||
}
|
||
}
|
||
|
||
if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
|
||
(Pfn1->OriginalPte.u.Soft.Transition == 1)) {
|
||
KeBugCheckEx (MEMORY_MANAGEMENT, 0x8889, 0,0,0);
|
||
}
|
||
#endif
|
||
|
||
//
|
||
// Check to ensure the reference count for the page is zero.
|
||
//
|
||
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
||
|
||
if (Pfn1->u3.e1.Rom == 1) {
|
||
|
||
//
|
||
// ROM pages do not go on transition lists and are not counted towards
|
||
// MmAvailablePages as they are not reusable. Migrate these pages
|
||
// into a separate list but keep the PageLocation as standby. Note
|
||
// it doesn't matter if the page is put at the head or the tail since
|
||
// it's not reusable.
|
||
//
|
||
|
||
ASSERT (XIPConfigured == TRUE);
|
||
ASSERT (Pfn1->u3.e1.Modified == 0);
|
||
|
||
ListHead = &MmRomPageListHead;
|
||
ListHead->Total += 1; // One more page on the list.
|
||
last = ListHead->Blink;
|
||
|
||
if (last != MM_EMPTY_LIST) {
|
||
Pfn2 = MI_PFN_ELEMENT (last);
|
||
Pfn2->u1.Flink = PageFrameIndex;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// List is empty, add the page to the ListHead.
|
||
//
|
||
|
||
ListHead->Flink = PageFrameIndex;
|
||
}
|
||
|
||
ListHead->Blink = PageFrameIndex;
|
||
Pfn1->u1.Flink = MM_EMPTY_LIST;
|
||
Pfn1->u2.Blink = last;
|
||
|
||
Pfn1->u3.e1.PageLocation = StandbyPageList;
|
||
|
||
return;
|
||
}
|
||
|
||
MmTransitionSharedPages += 1;
|
||
|
||
MmStandbyPageListHead.Total += 1; // One more page on the list.
|
||
|
||
ASSERT (MmTransitionPrivatePages + MmTransitionSharedPages == MmStandbyPageListHead.Total + MmModifiedPageListHead.Total + MmModifiedNoWritePageListHead.Total);
|
||
|
||
ListHead = &MmStandbyPageListHead;
|
||
|
||
first = ListHead->Flink;
|
||
if (first == MM_EMPTY_LIST) {
|
||
|
||
//
|
||
// List is empty add the page to the ListHead.
|
||
//
|
||
|
||
ListHead->Blink = PageFrameIndex;
|
||
}
|
||
else {
|
||
Pfn2 = MI_PFN_ELEMENT (first);
|
||
Pfn2->u2.Blink = PageFrameIndex;
|
||
}
|
||
|
||
ListHead->Flink = PageFrameIndex;
|
||
Pfn1->u2.Blink = MM_EMPTY_LIST;
|
||
Pfn1->u1.Flink = first;
|
||
Pfn1->u3.e1.PageLocation = StandbyPageList;
|
||
|
||
//
|
||
// If the page was placed on the free, standby or zeroed list,
|
||
// update the count of usable pages in the system. If the count
|
||
// transitions from 0 to 1, the event associated with available
|
||
// pages should become true.
|
||
//
|
||
|
||
MmAvailablePages += 1;
|
||
|
||
//
|
||
// A page has just become available, check to see if the
|
||
// page wait events should be signalled.
|
||
//
|
||
|
||
if (MmAvailablePages <= MM_HIGH_LIMIT) {
|
||
if (MmAvailablePages == MM_HIGH_LIMIT) {
|
||
KeSetEvent (&MmAvailablePagesEventHigh, 0, FALSE);
|
||
}
|
||
else if (MmAvailablePages == MM_MEDIUM_LIMIT) {
|
||
KeSetEvent (&MmAvailablePagesEventMedium, 0, FALSE);
|
||
}
|
||
else if (MmAvailablePages == MM_LOW_LIMIT) {
|
||
KeSetEvent (&MmAvailablePagesEvent, 0, FALSE);
|
||
}
|
||
}
|
||
|
||
//
|
||
// Signal applications if the freed page crosses a threshold.
|
||
//
|
||
|
||
if (MmAvailablePages == MmLowMemoryThreshold) {
|
||
KeClearEvent (MiLowMemoryEvent);
|
||
}
|
||
else if (MmAvailablePages == MmHighMemoryThreshold) {
|
||
KeSetEvent (MiHighMemoryEvent, 0, FALSE);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
PFN_NUMBER
|
||
FASTCALL
|
||
MiRemovePageFromList (
|
||
IN PMMPFNLIST ListHead
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure removes a page from the head of the specified list (free,
|
||
standby or zeroed).
|
||
|
||
This routine clears the flags word in the PFN database, hence the
|
||
PFN information for this page must be initialized.
|
||
|
||
Arguments:
|
||
|
||
ListHead - Supplies the list of the list in which to remove the
|
||
specified physical page.
|
||
|
||
Return Value:
|
||
|
||
The physical page number removed from the specified list.
|
||
|
||
Environment:
|
||
|
||
PFN lock held.
|
||
|
||
--*/
|
||
|
||
{
|
||
PMMCOLOR_TABLES ColorHead;
|
||
PFN_NUMBER PageFrameIndex;
|
||
PMMPFN Pfn1;
|
||
PMMPFN Pfn2;
|
||
ULONG Color;
|
||
MMLISTS ListName;
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
|
||
//
|
||
// If the specified list is empty return MM_EMPTY_LIST.
|
||
//
|
||
|
||
if (ListHead->Total == 0) {
|
||
KeBugCheckEx (PFN_LIST_CORRUPT, 1, (ULONG_PTR)ListHead, MmAvailablePages, 0);
|
||
}
|
||
|
||
ListName = ListHead->ListName;
|
||
ASSERT (ListName != ModifiedPageList);
|
||
|
||
//
|
||
// Decrement the count of pages on the list and remove the first
|
||
// page from the list.
|
||
//
|
||
|
||
ListHead->Total -= 1;
|
||
PageFrameIndex = ListHead->Flink;
|
||
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
||
|
||
if (PERFINFO_IS_GROUP_ON(PERF_MEMORY)) {
|
||
MiLogPfnInformation (Pfn1, PERFINFO_LOG_TYPE_REMOVEPAGEFROMLIST);
|
||
}
|
||
|
||
ListHead->Flink = Pfn1->u1.Flink;
|
||
|
||
//
|
||
// Zero the flink and blink in the PFN database element.
|
||
//
|
||
|
||
Pfn1->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
||
Pfn1->u2.Blink = 0;
|
||
|
||
//
|
||
// If the last page was removed (the ListHead->Flink is now
|
||
// MM_EMPTY_LIST) make the Listhead->Blink MM_EMPTY_LIST as well.
|
||
//
|
||
|
||
if (ListHead->Flink != MM_EMPTY_LIST) {
|
||
|
||
//
|
||
// Make the PFN element blink point to MM_EMPTY_LIST signifying this
|
||
// is the first page in the list.
|
||
//
|
||
|
||
Pfn2 = MI_PFN_ELEMENT (ListHead->Flink);
|
||
Pfn2->u2.Blink = MM_EMPTY_LIST;
|
||
}
|
||
else {
|
||
ListHead->Blink = MM_EMPTY_LIST;
|
||
}
|
||
|
||
//
|
||
// Check to see if we now have one less page available.
|
||
//
|
||
|
||
if (ListName <= StandbyPageList) {
|
||
MmAvailablePages -= 1;
|
||
|
||
if (ListName == StandbyPageList) {
|
||
|
||
//
|
||
// This page is currently in transition, restore the PTE to
|
||
// its original contents so this page can be reused.
|
||
//
|
||
|
||
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn1);
|
||
MiRestoreTransitionPte (PageFrameIndex);
|
||
}
|
||
|
||
if (MmAvailablePages < MmMinimumFreePages) {
|
||
|
||
//
|
||
// Obtain free pages.
|
||
//
|
||
|
||
MiObtainFreePages();
|
||
}
|
||
}
|
||
|
||
ASSERT ((PageFrameIndex != 0) &&
|
||
(PageFrameIndex <= MmHighestPhysicalPage) &&
|
||
(PageFrameIndex >= MmLowestPhysicalPage));
|
||
|
||
//
|
||
// Zero the PFN flags longword.
|
||
//
|
||
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
Color = Pfn1->u3.e1.PageColor;
|
||
ASSERT (Pfn1->u3.e1.RemovalRequested == 0);
|
||
ASSERT (Pfn1->u3.e1.Rom == 0);
|
||
Pfn1->u3.e2.ShortFlags = 0;
|
||
Pfn1->u3.e1.PageColor = Color;
|
||
Pfn1->u3.e1.CacheAttribute = MiNotMapped;
|
||
|
||
if (ListName <= FreePageList) {
|
||
|
||
//
|
||
// Update the color lists.
|
||
//
|
||
|
||
Color = MI_GET_COLOR_FROM_LIST_ENTRY(PageFrameIndex, Pfn1);
|
||
ColorHead = &MmFreePagesByColor[ListName][Color];
|
||
ASSERT (ColorHead->Flink == PageFrameIndex);
|
||
ColorHead->Flink = (PFN_NUMBER) Pfn1->OriginalPte.u.Long;
|
||
ASSERT (ColorHead->Count >= 1);
|
||
ColorHead->Count -= 1;
|
||
}
|
||
|
||
return PageFrameIndex;
|
||
}
|
||
|
||
VOID
|
||
FASTCALL
|
||
MiUnlinkPageFromList (
|
||
IN PMMPFN Pfn
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure removes a page from the middle of a list. This is
|
||
designed for the faulting of transition pages from the standby and
|
||
modified list and making them active and valid again.
|
||
|
||
Arguments:
|
||
|
||
Pfn - Supplies a pointer to the PFN database element for the physical
|
||
page to remove from the list.
|
||
|
||
Return Value:
|
||
|
||
none.
|
||
|
||
Environment:
|
||
|
||
Must be holding the PFN database lock with APCs disabled.
|
||
|
||
--*/
|
||
|
||
{
|
||
PMMPFNLIST ListHead;
|
||
PFN_NUMBER Previous;
|
||
PFN_NUMBER Next;
|
||
PMMPFN Pfn2;
|
||
MMLISTS ListName;
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
|
||
PERFINFO_UNLINKPAGE((ULONG_PTR)(Pfn - MmPfnDatabase), Pfn->u3.e1.PageLocation);
|
||
|
||
//
|
||
// Page not on standby or modified list, check to see if the
|
||
// page is currently being written by the modified page
|
||
// writer, if so, just return this page. The reference
|
||
// count for the page will be incremented, so when the modified
|
||
// page write completes, the page will not be put back on
|
||
// the list, rather, it will remain active and valid.
|
||
//
|
||
|
||
if (Pfn->u3.e2.ReferenceCount > 0) {
|
||
|
||
//
|
||
// The page was not on any "transition lists", check to see
|
||
// if this has I/O in progress.
|
||
//
|
||
|
||
if (Pfn->u2.ShareCount == 0) {
|
||
#if DBG
|
||
if (MmDebug & MM_DBG_PAGE_IN_LIST) {
|
||
DbgPrint("unlinking page not in list...\n");
|
||
MiFormatPfn(Pfn);
|
||
}
|
||
#endif
|
||
return;
|
||
}
|
||
KdPrint(("MM:attempt to remove page from wrong page list\n"));
|
||
KeBugCheckEx (PFN_LIST_CORRUPT,
|
||
2,
|
||
Pfn - MmPfnDatabase,
|
||
MmHighestPhysicalPage,
|
||
Pfn->u3.e2.ReferenceCount);
|
||
}
|
||
|
||
ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
|
||
|
||
//
|
||
// Must not remove pages from free or zeroed without updating
|
||
// the colored lists.
|
||
//
|
||
|
||
ListName = ListHead->ListName;
|
||
ASSERT (ListName >= StandbyPageList);
|
||
|
||
//
|
||
// If memory mirroring is in progress, any additions or removals to the
|
||
// free, zeroed, standby, modified or modified-no-write lists must
|
||
// update the bitmap.
|
||
//
|
||
|
||
if (MiMirroringActive == TRUE) {
|
||
RtlSetBit (MiMirrorBitMap2, (ULONG)(Pfn - MmPfnDatabase));
|
||
}
|
||
|
||
ASSERT (Pfn->u3.e1.CacheAttribute == MiCached);
|
||
|
||
if (ListHead == &MmStandbyPageListHead) {
|
||
if (Pfn->u3.e1.Rom == 1) {
|
||
ASSERT (XIPConfigured == TRUE);
|
||
ASSERT (Pfn->u3.e1.Modified == 0);
|
||
|
||
ListHead = &MmRomPageListHead;
|
||
|
||
//
|
||
// Deliberately switch to an invalid page listname so the checks
|
||
// below do not alter available pages, etc.
|
||
//
|
||
|
||
ListName = ActiveAndValid;
|
||
}
|
||
}
|
||
else if ((ListHead == &MmModifiedPageListHead) &&
|
||
(Pfn->OriginalPte.u.Soft.Prototype == 0)) {
|
||
|
||
//
|
||
// On MIPS R4000 modified pages destined for the paging file are
|
||
// kept on separate lists which group pages of the same color
|
||
// together.
|
||
//
|
||
|
||
//
|
||
// This page is destined for the paging file (not
|
||
// a mapped file). Change the list head to the
|
||
// appropriate colored list head.
|
||
//
|
||
|
||
ListHead->Total -= 1;
|
||
MmTotalPagesForPagingFile -= 1;
|
||
#if MM_MAXIMUM_NUMBER_OF_COLORS > 1
|
||
ListHead = &MmModifiedPageListByColor [Pfn->u3.e1.PageColor];
|
||
#else
|
||
ListHead = &MmModifiedPageListByColor [0];
|
||
#endif
|
||
ASSERT (ListHead->ListName == ListName);
|
||
}
|
||
|
||
ASSERT (Pfn->u3.e1.WriteInProgress == 0);
|
||
ASSERT (Pfn->u3.e1.ReadInProgress == 0);
|
||
ASSERT (ListHead->Total != 0);
|
||
|
||
Next = Pfn->u1.Flink;
|
||
Pfn->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
||
Previous = Pfn->u2.Blink;
|
||
Pfn->u2.Blink = 0;
|
||
|
||
if (Next != MM_EMPTY_LIST) {
|
||
Pfn2 = MI_PFN_ELEMENT(Next);
|
||
Pfn2->u2.Blink = Previous;
|
||
}
|
||
else {
|
||
ListHead->Blink = Previous;
|
||
}
|
||
|
||
if (Previous != MM_EMPTY_LIST) {
|
||
Pfn2 = MI_PFN_ELEMENT(Previous);
|
||
Pfn2->u1.Flink = Next;
|
||
}
|
||
else {
|
||
ListHead->Flink = Next;
|
||
}
|
||
|
||
ListHead->Total -= 1;
|
||
|
||
//
|
||
// Check to see if we now have one less page available.
|
||
//
|
||
|
||
if (ListName <= StandbyPageList) {
|
||
MmAvailablePages -= 1;
|
||
|
||
if (ListName == StandbyPageList) {
|
||
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn);
|
||
}
|
||
|
||
if (MmAvailablePages < MmMinimumFreePages) {
|
||
|
||
//
|
||
// Obtain free pages.
|
||
//
|
||
|
||
MiObtainFreePages();
|
||
|
||
}
|
||
}
|
||
else if (ListName == ModifiedPageList || ListName == ModifiedNoWritePageList) {
|
||
MI_TALLY_TRANSITION_PAGE_REMOVAL (Pfn);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
MiUnlinkFreeOrZeroedPage (
|
||
IN PFN_NUMBER Page
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure removes a page from the middle of a list. This is
|
||
designed for the removing of free or zeroed pages from the middle of
|
||
their lists.
|
||
|
||
Arguments:
|
||
|
||
Page - Supplies a page frame index to remove from the list.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Environment:
|
||
|
||
Must be holding the PFN database lock with APCs disabled.
|
||
|
||
--*/
|
||
|
||
{
|
||
PMMPFNLIST ListHead;
|
||
PFN_NUMBER Previous;
|
||
PFN_NUMBER Next;
|
||
PMMPFN Pfn2;
|
||
PMMPFN Pfn;
|
||
ULONG Color;
|
||
PMMCOLOR_TABLES ColorHead;
|
||
MMLISTS ListName;
|
||
|
||
Pfn = MI_PFN_ELEMENT (Page);
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
|
||
ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
|
||
ListName = ListHead->ListName;
|
||
ASSERT (ListHead->Total != 0);
|
||
ListHead->Total -= 1;
|
||
|
||
ASSERT (ListName <= FreePageList);
|
||
ASSERT (Pfn->u3.e1.WriteInProgress == 0);
|
||
ASSERT (Pfn->u3.e1.ReadInProgress == 0);
|
||
ASSERT (Pfn->u3.e1.CacheAttribute == MiNotMapped);
|
||
|
||
//
|
||
// If memory mirroring is in progress, any removals from the
|
||
// free, zeroed, standby, modified or modified-no-write lists that
|
||
// isn't immediately re-inserting into one of these 5 lists (WITHOUT
|
||
// modifying the page contents) must update the bitmap.
|
||
//
|
||
|
||
if (MiMirroringActive == TRUE) {
|
||
RtlSetBit (MiMirrorBitMap2, (ULONG)Page);
|
||
}
|
||
|
||
PERFINFO_UNLINKFREEPAGE (Page, Pfn->u3.e1.PageLocation);
|
||
|
||
Next = Pfn->u1.Flink;
|
||
Pfn->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
||
Previous = Pfn->u2.Blink;
|
||
Pfn->u2.Blink = 0;
|
||
|
||
if (Next != MM_EMPTY_LIST) {
|
||
Pfn2 = MI_PFN_ELEMENT(Next);
|
||
Pfn2->u2.Blink = Previous;
|
||
}
|
||
else {
|
||
ListHead->Blink = Previous;
|
||
}
|
||
|
||
if (Previous == MM_EMPTY_LIST) {
|
||
ListHead->Flink = Next;
|
||
}
|
||
else {
|
||
Pfn2 = MI_PFN_ELEMENT(Previous);
|
||
Pfn2->u1.Flink = Next;
|
||
}
|
||
|
||
//
|
||
// We are removing a page from the middle of the free or zeroed page list.
|
||
// The secondary color tables must be updated at this time.
|
||
//
|
||
|
||
Color = MI_GET_COLOR_FROM_LIST_ENTRY(Page, Pfn);
|
||
|
||
//
|
||
// Walk down the list and remove the page.
|
||
//
|
||
|
||
ColorHead = &MmFreePagesByColor[ListName][Color];
|
||
Next = ColorHead->Flink;
|
||
if (Next == Page) {
|
||
ColorHead->Flink = (PFN_NUMBER) Pfn->OriginalPte.u.Long;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// Walk the list to find the parent.
|
||
//
|
||
|
||
do {
|
||
Pfn2 = MI_PFN_ELEMENT (Next);
|
||
Next = (PFN_NUMBER) Pfn2->OriginalPte.u.Long;
|
||
if (Page == Next) {
|
||
Pfn2->OriginalPte.u.Long = Pfn->OriginalPte.u.Long;
|
||
if ((PFN_NUMBER) Pfn->OriginalPte.u.Long == MM_EMPTY_LIST) {
|
||
ColorHead->Blink = Pfn2;
|
||
}
|
||
break;
|
||
}
|
||
} while (TRUE);
|
||
}
|
||
|
||
ASSERT (ColorHead->Count >= 1);
|
||
ColorHead->Count -= 1;
|
||
|
||
//
|
||
// Decrement availability count.
|
||
//
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
if (KeNumberNodes > 1) {
|
||
MI_NODE_FROM_COLOR(Color)->FreeCount[ListName]--;
|
||
}
|
||
|
||
#endif
|
||
|
||
MmAvailablePages -= 1;
|
||
|
||
if (MmAvailablePages < MmMinimumFreePages) {
|
||
|
||
//
|
||
// Obtain free pages.
|
||
//
|
||
|
||
MiObtainFreePages();
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
|
||
ULONG
|
||
FASTCALL
|
||
MiEnsureAvailablePageOrWait (
|
||
IN PEPROCESS Process,
|
||
IN PVOID VirtualAddress
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure ensures that a physical page is available on
|
||
the zeroed, free or standby list such that the next call the remove a
|
||
page absolutely will not block. This is necessary as blocking would
|
||
require a wait which could cause a deadlock condition.
|
||
|
||
If a page is available the function returns immediately with a value
|
||
of FALSE indicating no wait operation was performed. If no physical
|
||
page is available, the thread enters a wait state and the function
|
||
returns the value TRUE when the wait operation completes.
|
||
|
||
Arguments:
|
||
|
||
Process - Supplies a pointer to the current process if, and only if,
|
||
the working set mutex is held currently held and should
|
||
be released if a wait operation is issued. Supplies
|
||
the value NULL otherwise.
|
||
|
||
VirtualAddress - Supplies the virtual address for the faulting page.
|
||
If the value is NULL, the page is treated as a
|
||
user mode address.
|
||
|
||
Return Value:
|
||
|
||
FALSE - if a page was immediately available.
|
||
TRUE - if a wait operation occurred before a page became available.
|
||
|
||
|
||
Environment:
|
||
|
||
Must be holding the PFN database lock with APCs disabled.
|
||
|
||
--*/
|
||
|
||
{
|
||
PVOID Event;
|
||
NTSTATUS Status;
|
||
KIRQL OldIrql;
|
||
KIRQL Ignore;
|
||
ULONG Limit;
|
||
ULONG Relock;
|
||
PFN_NUMBER StrandedPages;
|
||
LOGICAL WsHeldSafe;
|
||
PMMPFN Pfn1;
|
||
PMMPFN EndPfn;
|
||
LARGE_INTEGER WaitBegin;
|
||
LARGE_INTEGER WaitEnd;
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
|
||
if (MmAvailablePages >= MM_HIGH_LIMIT) {
|
||
|
||
//
|
||
// Pages are available.
|
||
//
|
||
|
||
return FALSE;
|
||
}
|
||
|
||
//
|
||
// If this fault is for paged pool (or pagable kernel space,
|
||
// including page table pages), let it use the last page.
|
||
//
|
||
|
||
#if defined(_IA64_)
|
||
if (MI_IS_SYSTEM_ADDRESS(VirtualAddress) ||
|
||
(MI_IS_HYPER_SPACE_ADDRESS(VirtualAddress))) {
|
||
#else
|
||
if (((PMMPTE)VirtualAddress > MiGetPteAddress(HYPER_SPACE)) ||
|
||
((VirtualAddress > MM_HIGHEST_USER_ADDRESS) &&
|
||
(VirtualAddress < (PVOID)PTE_BASE))) {
|
||
#endif
|
||
|
||
//
|
||
// This fault is in the system, use 1 page as the limit.
|
||
//
|
||
|
||
if (MmAvailablePages >= MM_LOW_LIMIT) {
|
||
|
||
//
|
||
// Pages are available.
|
||
//
|
||
|
||
return FALSE;
|
||
}
|
||
|
||
Limit = MM_LOW_LIMIT;
|
||
Event = (PVOID)&MmAvailablePagesEvent;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// If this thread has explicitly disabled APCs (FsRtlEnterFileSystem
|
||
// does this), then it may be holding resources or mutexes that may in
|
||
// turn be blocking memory making threads from making progress.
|
||
//
|
||
// Likewise give system threads a free pass as they may be worker
|
||
// threads processing potentially blocking items drivers have queued.
|
||
//
|
||
|
||
if ((IS_SYSTEM_THREAD(PsGetCurrentThread())) ||
|
||
(KeGetCurrentThread()->KernelApcDisable != 0)) {
|
||
|
||
if (MmAvailablePages >= MM_MEDIUM_LIMIT) {
|
||
|
||
//
|
||
// Pages are available.
|
||
//
|
||
|
||
return FALSE;
|
||
}
|
||
|
||
Limit = MM_MEDIUM_LIMIT;
|
||
Event = (PVOID) &MmAvailablePagesEventMedium;
|
||
}
|
||
else {
|
||
Limit = MM_HIGH_LIMIT;
|
||
Event = (PVOID) &MmAvailablePagesEventHigh;
|
||
}
|
||
}
|
||
|
||
//
|
||
// Initializing WsHeldSafe is not needed for
|
||
// correctness but without it the compiler cannot compile this code
|
||
// W4 to check for use of uninitialized variables.
|
||
//
|
||
|
||
WsHeldSafe = FALSE;
|
||
|
||
while (MmAvailablePages < Limit) {
|
||
|
||
KeClearEvent ((PKEVENT)Event);
|
||
|
||
UNLOCK_PFN (APC_LEVEL);
|
||
|
||
Relock = FALSE;
|
||
|
||
if (Process == HYDRA_PROCESS) {
|
||
UNLOCK_SESSION_SPACE_WS (APC_LEVEL);
|
||
}
|
||
else if (Process != NULL) {
|
||
|
||
//
|
||
// The working set lock may have been acquired safely or unsafely
|
||
// by our caller. Handle both cases here and below.
|
||
//
|
||
|
||
UNLOCK_WS_REGARDLESS (Process, WsHeldSafe);
|
||
}
|
||
else {
|
||
if (MmSystemLockOwner == PsGetCurrentThread()) {
|
||
UNLOCK_SYSTEM_WS (APC_LEVEL);
|
||
Relock = TRUE;
|
||
}
|
||
}
|
||
|
||
KiQueryInterruptTime(&WaitBegin);
|
||
|
||
//
|
||
// Wait 7 minutes for pages to become available.
|
||
//
|
||
|
||
Status = KeWaitForSingleObject(Event,
|
||
WrFreePage,
|
||
KernelMode,
|
||
FALSE,
|
||
(PLARGE_INTEGER)&MmSevenMinutes);
|
||
|
||
if (Status == STATUS_TIMEOUT) {
|
||
|
||
KiQueryInterruptTime(&WaitEnd);
|
||
|
||
if (MmSystemShutdown != 0) {
|
||
|
||
//
|
||
// Because applications are not terminated and drivers are
|
||
// not unloaded, they can continue to access pages even after
|
||
// the modified writer has terminated. This can cause the
|
||
// system to run out of pages since the pagefile(s) cannot be
|
||
// used.
|
||
//
|
||
|
||
KeBugCheckEx (DISORDERLY_SHUTDOWN,
|
||
MmModifiedPageListHead.Total,
|
||
MmTotalPagesForPagingFile,
|
||
(MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT) - MmAllocatedNonPagedPool,
|
||
MmSystemShutdown);
|
||
}
|
||
|
||
//
|
||
// See how many transition pages have nonzero reference counts as
|
||
// these indicate drivers that aren't unlocking the pages in their
|
||
// MDLs.
|
||
//
|
||
|
||
Limit = 0;
|
||
StrandedPages = 0;
|
||
|
||
do {
|
||
|
||
Pfn1 = MI_PFN_ELEMENT (MmPhysicalMemoryBlock->Run[Limit].BasePage);
|
||
EndPfn = Pfn1 + MmPhysicalMemoryBlock->Run[Limit].PageCount;
|
||
|
||
while (Pfn1 < EndPfn) {
|
||
if ((Pfn1->u3.e1.PageLocation == TransitionPage) &&
|
||
(Pfn1->u3.e2.ReferenceCount != 0)) {
|
||
StrandedPages += 1;
|
||
}
|
||
Pfn1 += 1;
|
||
}
|
||
Limit += 1;
|
||
|
||
} while (Limit != MmPhysicalMemoryBlock->NumberOfRuns);
|
||
|
||
//
|
||
// This bugcheck can occur for the following reasons:
|
||
//
|
||
// A driver has blocked, deadlocking the modified or mapped
|
||
// page writers. Examples of this include mutex deadlocks or
|
||
// accesses to paged out memory in filesystem drivers, filter
|
||
// drivers, etc. This indicates a driver bug.
|
||
//
|
||
// The storage driver(s) are not processing requests. Examples
|
||
// of this are stranded queues, non-responding drives, etc. This
|
||
// indicates a driver bug.
|
||
//
|
||
// Not enough pool is available for the storage stack to write out
|
||
// modified pages. This indicates a driver bug.
|
||
//
|
||
// A high priority realtime thread has starved the balance set
|
||
// manager from trimming pages and/or starved the modified writer
|
||
// from writing them out. This indicates a bug in the component
|
||
// that created this thread.
|
||
//
|
||
|
||
StrandedPages &= ~3;
|
||
StrandedPages |= MmSystemShutdown;
|
||
|
||
if (KdDebuggerNotPresent) {
|
||
if (MmTotalPagesForPagingFile >= (MmModifiedPageListHead.Total >> 2)) {
|
||
KeBugCheckEx (NO_PAGES_AVAILABLE,
|
||
MmModifiedPageListHead.Total,
|
||
MmTotalPagesForPagingFile,
|
||
(MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT) - MmAllocatedNonPagedPool,
|
||
StrandedPages);
|
||
}
|
||
KeBugCheckEx (DIRTY_MAPPED_PAGES_CONGESTION,
|
||
MmModifiedPageListHead.Total,
|
||
MmTotalPagesForPagingFile,
|
||
(MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT) - MmAllocatedNonPagedPool,
|
||
StrandedPages);
|
||
}
|
||
|
||
DbgPrint ("MmEnsureAvailablePageOrWait: 7 min timeout %x %x %x %x\n", WaitEnd.HighPart, WaitEnd.LowPart, WaitBegin.HighPart, WaitBegin.LowPart);
|
||
|
||
DbgPrint ("Without a debugger attached, the following bugcheck would have occurred.\n");
|
||
if (MmTotalPagesForPagingFile >= (MmModifiedPageListHead.Total >> 2)) {
|
||
DbgPrint ("%3lx ", NO_PAGES_AVAILABLE);
|
||
}
|
||
else {
|
||
DbgPrint ("%3lxp ", DIRTY_MAPPED_PAGES_CONGESTION);
|
||
}
|
||
|
||
DbgPrint ("%p %p %p %p\n",
|
||
MmModifiedPageListHead.Total,
|
||
MmTotalPagesForPagingFile,
|
||
(MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT) - MmAllocatedNonPagedPool,
|
||
StrandedPages);
|
||
|
||
//
|
||
// Pop into the debugger (even on free builds) to determine
|
||
// the cause of the starvation and march on.
|
||
//
|
||
|
||
DbgBreakPoint ();
|
||
}
|
||
|
||
if (Process == HYDRA_PROCESS) {
|
||
LOCK_SESSION_SPACE_WS (Ignore, PsGetCurrentThread ());
|
||
}
|
||
else if (Process != NULL) {
|
||
|
||
//
|
||
// The working set lock may have been acquired safely or unsafely
|
||
// by our caller. Reacquire it in the same manner our caller did.
|
||
//
|
||
|
||
LOCK_WS_REGARDLESS (Process, WsHeldSafe);
|
||
}
|
||
else {
|
||
if (Relock) {
|
||
LOCK_SYSTEM_WS (Ignore, PsGetCurrentThread ());
|
||
}
|
||
}
|
||
|
||
LOCK_PFN (OldIrql);
|
||
}
|
||
|
||
return TRUE;
|
||
}
|
||
|
||
|
||
PFN_NUMBER
|
||
FASTCALL
|
||
MiRemoveZeroPage (
|
||
IN ULONG Color
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure removes a zero page from either the zeroed, free
|
||
or standby lists (in that order). If no pages exist on the zeroed
|
||
or free list a transition page is removed from the standby list
|
||
and the PTE (may be a prototype PTE) which refers to this page is
|
||
changed from transition back to its original contents.
|
||
|
||
If the page is not obtained from the zeroed list, it is zeroed.
|
||
|
||
Arguments:
|
||
|
||
Color - Supplies the page color for which this page is destined.
|
||
This is used for checking virtual address alignments to
|
||
determine if the D cache needs flushing before the page
|
||
can be reused.
|
||
|
||
The above was true when we were concerned about caches
|
||
which are virtually indexed (ie MIPS). Today we
|
||
are more concerned that we get a good usage spread across
|
||
the L2 caches of most machines. These caches are physically
|
||
indexed. By gathering pages that would have the same
|
||
index to the same color, then maximizing the color spread,
|
||
we maximize the effective use of the caches.
|
||
|
||
This has been extended for NUMA machines. The high part
|
||
of the color gives the node color (basically node number).
|
||
If we cannot allocate a page of the requested color, we
|
||
try to allocate a page on the same node before taking a
|
||
page from a different node.
|
||
|
||
Return Value:
|
||
|
||
The physical page number removed from the specified list.
|
||
|
||
Environment:
|
||
|
||
Must be holding the PFN database lock with APCs disabled.
|
||
|
||
--*/
|
||
|
||
{
|
||
PFN_NUMBER Page;
|
||
PMMPFN Pfn1;
|
||
PMMCOLOR_TABLES FreePagesByColor;
|
||
#if MI_BARRIER_SUPPORTED
|
||
ULONG BarrierStamp;
|
||
#endif
|
||
#if defined(MI_MULTINODE)
|
||
PKNODE Node;
|
||
ULONG NodeColor;
|
||
ULONG OriginalColor;
|
||
#endif
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
ASSERT(MmAvailablePages != 0);
|
||
|
||
FreePagesByColor = MmFreePagesByColor[ZeroedPageList];
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
//
|
||
// Initializing Node is not needed for correctness, but without it
|
||
// the compiler cannot compile this code W4 to check for use of
|
||
// uninitialized variables.
|
||
//
|
||
|
||
Node = NULL;
|
||
|
||
NodeColor = Color & ~MmSecondaryColorMask;
|
||
OriginalColor = Color;
|
||
|
||
if (KeNumberNodes > 1) {
|
||
Node = MI_NODE_FROM_COLOR(Color);
|
||
}
|
||
|
||
do {
|
||
|
||
#endif
|
||
|
||
//
|
||
// Attempt to remove a page from the zeroed page list. If a page
|
||
// is available, then remove it and return its page frame index.
|
||
// Otherwise, attempt to remove a page from the free page list or
|
||
// the standby list.
|
||
//
|
||
// N.B. It is not necessary to change page colors even if the old
|
||
// color is not equal to the new color. The zero page thread
|
||
// ensures that all zeroed pages are removed from all caches.
|
||
//
|
||
|
||
ASSERT (Color < MmSecondaryColors);
|
||
Page = FreePagesByColor[Color].Flink;
|
||
|
||
if (Page != MM_EMPTY_LIST) {
|
||
|
||
//
|
||
// Remove the first entry on the zeroed by color list.
|
||
//
|
||
|
||
#if DBG
|
||
Pfn1 = MI_PFN_ELEMENT(Page);
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
#endif
|
||
ASSERT ((Pfn1->u3.e1.PageLocation == ZeroedPageList) ||
|
||
((Pfn1->u3.e1.PageLocation == FreePageList) &&
|
||
(FreePagesByColor == MmFreePagesByColor[FreePageList])));
|
||
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
|
||
Page = MiRemovePageByColor (Page, Color);
|
||
|
||
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
if (FreePagesByColor != MmFreePagesByColor[ZeroedPageList]) {
|
||
goto ZeroPage;
|
||
}
|
||
|
||
#endif
|
||
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
|
||
return Page;
|
||
|
||
}
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
//
|
||
// If this is a multinode machine and there are zero
|
||
// pages on this node, select another color on this
|
||
// node in preference to random selection.
|
||
//
|
||
|
||
if (KeNumberNodes > 1) {
|
||
if (Node->FreeCount[ZeroedPageList] != 0) {
|
||
Color = ((Color + 1) & MmSecondaryColorMask) | NodeColor;
|
||
ASSERT(Color != OriginalColor);
|
||
continue;
|
||
}
|
||
|
||
//
|
||
// No previously zeroed page with the specified secondary
|
||
// color exists. Since this is a multinode machine, zero
|
||
// an available local free page now instead of allocating a
|
||
// zeroed page from another node below.
|
||
//
|
||
|
||
if (Node->FreeCount[FreePageList] != 0) {
|
||
if (FreePagesByColor != MmFreePagesByColor[FreePageList]) {
|
||
FreePagesByColor = MmFreePagesByColor[FreePageList];
|
||
Color = OriginalColor;
|
||
}
|
||
else {
|
||
Color = ((Color + 1) & MmSecondaryColorMask) | NodeColor;
|
||
ASSERT(Color != OriginalColor);
|
||
}
|
||
continue;
|
||
}
|
||
}
|
||
|
||
break;
|
||
} while (TRUE);
|
||
|
||
#endif
|
||
|
||
//
|
||
// No previously zeroed page with the specified secondary color exists.
|
||
// Try a zeroed page of any color.
|
||
//
|
||
|
||
Page = MmZeroedPageListHead.Flink;
|
||
if (Page != MM_EMPTY_LIST) {
|
||
#if DBG
|
||
Pfn1 = MI_PFN_ELEMENT(Page);
|
||
#endif
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
ASSERT (Pfn1->u3.e1.PageLocation == ZeroedPageList);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
|
||
Color = MI_GET_COLOR_FROM_LIST_ENTRY(Page, MI_PFN_ELEMENT(Page));
|
||
|
||
Page = MiRemovePageByColor (Page, Color);
|
||
|
||
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
|
||
return Page;
|
||
}
|
||
|
||
//
|
||
// No zeroed page of the primary color exists, try a free page of the
|
||
// secondary color. Note in the multinode case this has already been done
|
||
// above.
|
||
//
|
||
|
||
#if defined(MI_MULTINODE)
|
||
if (KeNumberNodes <= 1) {
|
||
#endif
|
||
FreePagesByColor = MmFreePagesByColor[FreePageList];
|
||
|
||
Page = FreePagesByColor[Color].Flink;
|
||
if (Page != MM_EMPTY_LIST) {
|
||
|
||
//
|
||
// Remove the first entry on the free list by color.
|
||
//
|
||
|
||
#if DBG
|
||
Pfn1 = MI_PFN_ELEMENT(Page);
|
||
#endif
|
||
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
|
||
Page = MiRemovePageByColor (Page, Color);
|
||
|
||
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
|
||
goto ZeroPage;
|
||
}
|
||
#if defined(MI_MULTINODE)
|
||
}
|
||
#endif
|
||
|
||
Page = MmFreePageListHead.Flink;
|
||
if (Page != MM_EMPTY_LIST) {
|
||
|
||
Color = MI_GET_COLOR_FROM_LIST_ENTRY(Page, MI_PFN_ELEMENT(Page));
|
||
#if DBG
|
||
Pfn1 = MI_PFN_ELEMENT(Page);
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
#endif
|
||
Page = MiRemovePageByColor (Page, Color);
|
||
|
||
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
goto ZeroPage;
|
||
}
|
||
|
||
ASSERT (MmZeroedPageListHead.Total == 0);
|
||
ASSERT (MmFreePageListHead.Total == 0);
|
||
|
||
//
|
||
// Remove a page from the standby list and restore the original
|
||
// contents of the PTE to free the last reference to the physical
|
||
// page.
|
||
//
|
||
|
||
ASSERT (MmStandbyPageListHead.Total != 0);
|
||
|
||
Page = MiRemovePageFromList (&MmStandbyPageListHead);
|
||
ASSERT ((MI_PFN_ELEMENT(Page))->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
ASSERT (MI_PFN_ELEMENT(Page)->u3.e1.CacheAttribute == MiNotMapped);
|
||
MiStandbyRemoved += 1;
|
||
|
||
//
|
||
// If memory mirroring is in progress, any removals from the
|
||
// free, zeroed, standby, modified or modified-no-write lists that
|
||
// isn't immediately re-inserting into one of these 5 lists (WITHOUT
|
||
// modifying the page contents) must update the bitmap.
|
||
//
|
||
|
||
if (MiMirroringActive == TRUE) {
|
||
RtlSetBit (MiMirrorBitMap2, (ULONG)Page);
|
||
}
|
||
|
||
//
|
||
// Zero the page removed from the free or standby list.
|
||
//
|
||
|
||
ZeroPage:
|
||
|
||
Pfn1 = MI_PFN_ELEMENT(Page);
|
||
|
||
MiZeroPhysicalPage (Page, 0);
|
||
|
||
#if MI_BARRIER_SUPPORTED
|
||
|
||
//
|
||
// Note the stamping must occur after the page is zeroed.
|
||
//
|
||
|
||
MI_BARRIER_STAMP_ZEROED_PAGE (&BarrierStamp);
|
||
Pfn1->u4.PteFrame = BarrierStamp;
|
||
|
||
#endif
|
||
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
|
||
return Page;
|
||
}
|
||
|
||
PFN_NUMBER
|
||
FASTCALL
|
||
MiRemoveAnyPage (
|
||
IN ULONG Color
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure removes a page from either the free, zeroed,
|
||
or standby lists (in that order). If no pages exist on the zeroed
|
||
or free list a transition page is removed from the standby list
|
||
and the PTE (may be a prototype PTE) which refers to this page is
|
||
changed from transition back to its original contents.
|
||
|
||
Note pages MUST exist to satisfy this request. The caller ensures this
|
||
by first calling MiEnsureAvailablePageOrWait.
|
||
|
||
Arguments:
|
||
|
||
Color - Supplies the page color for which this page is destined.
|
||
This is used for checking virtual address alignments to
|
||
determine if the D cache needs flushing before the page
|
||
can be reused.
|
||
|
||
The above was true when we were concerned about caches
|
||
which are virtually indexed. (eg MIPS). Today we
|
||
are more concerned that we get a good usage spread across
|
||
the L2 caches of most machines. These caches are physically
|
||
indexed. By gathering pages that would have the same
|
||
index to the same color, then maximizing the color spread,
|
||
we maximize the effective use of the caches.
|
||
|
||
This has been extended for NUMA machines. The high part
|
||
of the color gives the node color (basically node number).
|
||
If we cannot allocate a page of the requested color, we
|
||
try to allocate a page on the same node before taking a
|
||
page from a different node.
|
||
|
||
Return Value:
|
||
|
||
The physical page number removed from the specified list.
|
||
|
||
Environment:
|
||
|
||
Must be holding the PFN database lock with APCs disabled.
|
||
|
||
--*/
|
||
|
||
{
|
||
PFN_NUMBER Page;
|
||
#if DBG
|
||
PMMPFN Pfn1;
|
||
#endif
|
||
#if defined(MI_MULTINODE)
|
||
PKNODE Node;
|
||
ULONG NodeColor;
|
||
ULONG OriginalColor;
|
||
PFN_NUMBER LocalNodePagesAvailable;
|
||
#endif
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
ASSERT(MmAvailablePages != 0);
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
//
|
||
// Bias color to memory node. The assumption is that if memory
|
||
// of the correct color is not available on this node, it is
|
||
// better to choose memory of a different color if you can stay
|
||
// on this node.
|
||
//
|
||
|
||
LocalNodePagesAvailable = 0;
|
||
NodeColor = Color & ~MmSecondaryColorMask;
|
||
OriginalColor = Color;
|
||
|
||
if (KeNumberNodes > 1) {
|
||
Node = MI_NODE_FROM_COLOR(Color);
|
||
LocalNodePagesAvailable = (Node->FreeCount[ZeroedPageList] | Node->FreeCount[ZeroedPageList]);
|
||
}
|
||
|
||
do {
|
||
|
||
#endif
|
||
|
||
//
|
||
// Check the free page list, and if a page is available
|
||
// remove it and return its value.
|
||
//
|
||
|
||
ASSERT (Color < MmSecondaryColors);
|
||
if (MmFreePagesByColor[FreePageList][Color].Flink != MM_EMPTY_LIST) {
|
||
|
||
//
|
||
// Remove the first entry on the free by color list.
|
||
//
|
||
|
||
Page = MmFreePagesByColor[FreePageList][Color].Flink;
|
||
#if DBG
|
||
Pfn1 = MI_PFN_ELEMENT(Page);
|
||
#endif
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
|
||
Page = MiRemovePageByColor (Page, Color);
|
||
|
||
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
|
||
return Page;
|
||
}
|
||
|
||
//
|
||
// Try the zero page list by primary color.
|
||
//
|
||
|
||
if (MmFreePagesByColor[ZeroedPageList][Color].Flink
|
||
!= MM_EMPTY_LIST) {
|
||
|
||
//
|
||
// Remove the first entry on the zeroed by color list.
|
||
//
|
||
|
||
Page = MmFreePagesByColor[ZeroedPageList][Color].Flink;
|
||
#if DBG
|
||
Pfn1 = MI_PFN_ELEMENT(Page);
|
||
#endif
|
||
ASSERT (Pfn1->u3.e1.PageLocation == ZeroedPageList);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
|
||
Page = MiRemovePageByColor (Page, Color);
|
||
|
||
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
|
||
return Page;
|
||
}
|
||
|
||
//
|
||
// If this is a multinode machine and there are free
|
||
// pages on this node, select another color on this
|
||
// node in preference to random selection.
|
||
//
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
if (LocalNodePagesAvailable != 0) {
|
||
Color = ((Color + 1) & MmSecondaryColorMask) | NodeColor;
|
||
ASSERT(Color != OriginalColor);
|
||
continue;
|
||
}
|
||
|
||
break;
|
||
} while (TRUE);
|
||
|
||
#endif
|
||
|
||
//
|
||
// Check the free page list, and if a page is available
|
||
// remove it and return its value.
|
||
//
|
||
|
||
if (MmFreePageListHead.Flink != MM_EMPTY_LIST) {
|
||
Page = MmFreePageListHead.Flink;
|
||
Color = MI_GET_COLOR_FROM_LIST_ENTRY(Page, MI_PFN_ELEMENT(Page));
|
||
|
||
#if DBG
|
||
Pfn1 = MI_PFN_ELEMENT(Page);
|
||
#endif
|
||
ASSERT (Pfn1->u3.e1.PageLocation == FreePageList);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
|
||
Page = MiRemovePageByColor (Page, Color);
|
||
|
||
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
|
||
return Page;
|
||
}
|
||
ASSERT (MmFreePageListHead.Total == 0);
|
||
|
||
//
|
||
// Check the zeroed page list, and if a page is available
|
||
// remove it and return its value.
|
||
//
|
||
|
||
if (MmZeroedPageListHead.Flink != MM_EMPTY_LIST) {
|
||
Page = MmZeroedPageListHead.Flink;
|
||
Color = MI_GET_COLOR_FROM_LIST_ENTRY(Page, MI_PFN_ELEMENT(Page));
|
||
|
||
#if DBG
|
||
Pfn1 = MI_PFN_ELEMENT(Page);
|
||
#endif
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
Page = MiRemovePageByColor (Page, Color);
|
||
|
||
ASSERT (Pfn1 == MI_PFN_ELEMENT(Page));
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
|
||
return Page;
|
||
}
|
||
ASSERT (MmZeroedPageListHead.Total == 0);
|
||
|
||
//
|
||
// No pages exist on the free or zeroed list, use the standby list.
|
||
//
|
||
|
||
ASSERT(MmStandbyPageListHead.Total != 0);
|
||
|
||
Page = MiRemovePageFromList (&MmStandbyPageListHead);
|
||
ASSERT ((MI_PFN_ELEMENT(Page))->u4.PteFrame != MI_MAGIC_AWE_PTEFRAME);
|
||
ASSERT ((MI_PFN_ELEMENT(Page))->u3.e1.CacheAttribute == MiNotMapped);
|
||
MiStandbyRemoved += 1;
|
||
|
||
//
|
||
// If memory mirroring is in progress, any removals from the
|
||
// free, zeroed, standby, modified or modified-no-write lists that
|
||
// isn't immediately re-inserting into one of these 5 lists (WITHOUT
|
||
// modifying the page contents) must update the bitmap.
|
||
//
|
||
|
||
if (MiMirroringActive == TRUE) {
|
||
RtlSetBit (MiMirrorBitMap2, (ULONG)Page);
|
||
}
|
||
|
||
MI_CHECK_PAGE_ALIGNMENT(Page, Color & MM_COLOR_MASK);
|
||
#if DBG
|
||
Pfn1 = MI_PFN_ELEMENT (Page);
|
||
#endif
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
|
||
return Page;
|
||
}
|
||
|
||
|
||
PFN_NUMBER
|
||
FASTCALL
|
||
MiRemovePageByColor (
|
||
IN PFN_NUMBER Page,
|
||
IN ULONG Color
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure removes a page from the middle of the free or
|
||
zeroed page list.
|
||
|
||
Arguments:
|
||
|
||
Page - Supplies the physical page number to unlink from the list.
|
||
|
||
Color - Supplies the page color for which this page is destined.
|
||
This is used for checking virtual address alignments to
|
||
determine if the D cache needs flushing before the page
|
||
can be reused.
|
||
|
||
Return Value:
|
||
|
||
The page frame number that was unlinked (always equal to the one
|
||
passed in, but returned so the caller's fastcall sequences save
|
||
extra register pushes and pops.
|
||
|
||
Environment:
|
||
|
||
Must be holding the PFN database lock with APCs disabled.
|
||
|
||
--*/
|
||
|
||
{
|
||
PMMPFNLIST ListHead;
|
||
PMMPFNLIST PrimaryListHead;
|
||
PFN_NUMBER Previous;
|
||
PFN_NUMBER Next;
|
||
PMMPFN Pfn1;
|
||
PMMPFN Pfn2;
|
||
ULONG NodeColor;
|
||
MMLISTS ListName;
|
||
PMMCOLOR_TABLES ColorHead;
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
|
||
Pfn1 = MI_PFN_ELEMENT (Page);
|
||
NodeColor = Pfn1->u3.e1.PageColor;
|
||
|
||
#if defined(MI_MULTINODE)
|
||
|
||
ASSERT (NodeColor == (Color >> MmSecondaryColorNodeShift));
|
||
|
||
#endif
|
||
|
||
if (PERFINFO_IS_GROUP_ON(PERF_MEMORY)) {
|
||
MiLogPfnInformation (Pfn1, PERFINFO_LOG_TYPE_REMOVEPAGEBYCOLOR);
|
||
}
|
||
|
||
//
|
||
// If memory mirroring is in progress, any additions or removals to the
|
||
// free, zeroed, standby, modified or modified-no-write lists must
|
||
// update the bitmap.
|
||
//
|
||
|
||
if (MiMirroringActive == TRUE) {
|
||
RtlSetBit (MiMirrorBitMap2, (ULONG)Page);
|
||
}
|
||
|
||
ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
|
||
ListName = ListHead->ListName;
|
||
|
||
ListHead->Total -= 1;
|
||
|
||
PrimaryListHead = ListHead;
|
||
|
||
Next = Pfn1->u1.Flink;
|
||
Pfn1->u1.Flink = 0; // Assumes Flink width is >= WsIndex width
|
||
Previous = Pfn1->u2.Blink;
|
||
Pfn1->u2.Blink = 0;
|
||
|
||
if (Next == MM_EMPTY_LIST) {
|
||
PrimaryListHead->Blink = Previous;
|
||
}
|
||
else {
|
||
Pfn2 = MI_PFN_ELEMENT(Next);
|
||
Pfn2->u2.Blink = Previous;
|
||
}
|
||
|
||
if (Previous == MM_EMPTY_LIST) {
|
||
PrimaryListHead->Flink = Next;
|
||
}
|
||
else {
|
||
Pfn2 = MI_PFN_ELEMENT(Previous);
|
||
Pfn2->u1.Flink = Next;
|
||
}
|
||
|
||
ASSERT (Pfn1->u3.e1.RemovalRequested == 0);
|
||
|
||
//
|
||
// Zero the flags longword, but keep the color and cache information.
|
||
//
|
||
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiNotMapped);
|
||
ASSERT (Pfn1->u3.e1.Rom == 0);
|
||
Pfn1->u3.e2.ShortFlags = 0;
|
||
Pfn1->u3.e1.PageColor = NodeColor;
|
||
Pfn1->u3.e1.CacheAttribute = MiNotMapped;
|
||
|
||
//
|
||
// Update the color lists.
|
||
//
|
||
|
||
ASSERT (Color < MmSecondaryColors);
|
||
ColorHead = &MmFreePagesByColor[ListName][Color];
|
||
ColorHead->Flink = (PFN_NUMBER) Pfn1->OriginalPte.u.Long;
|
||
ASSERT (ColorHead->Count >= 1);
|
||
ColorHead->Count -= 1;
|
||
|
||
//
|
||
// Note that we now have one less page available.
|
||
//
|
||
|
||
#if defined(MI_MULTINODE)
|
||
if (KeNumberNodes > 1) {
|
||
KeNodeBlock[NodeColor]->FreeCount[ListName]--;
|
||
}
|
||
#endif
|
||
|
||
MmAvailablePages -= 1;
|
||
|
||
if (MmAvailablePages < MmMinimumFreePages) {
|
||
|
||
//
|
||
// Obtain free pages.
|
||
//
|
||
|
||
MiObtainFreePages();
|
||
}
|
||
|
||
return Page;
|
||
}
|
||
|
||
|
||
VOID
|
||
FASTCALL
|
||
MiInsertFrontModifiedNoWrite (
|
||
IN PFN_NUMBER PageFrameIndex
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure inserts a page at the FRONT of the modified no write list.
|
||
|
||
Arguments:
|
||
|
||
PageFrameIndex - Supplies the physical page number to insert in the list.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Environment:
|
||
|
||
Must be holding the PFN database lock with APCs disabled.
|
||
|
||
--*/
|
||
|
||
{
|
||
PFN_NUMBER first;
|
||
PMMPFN Pfn1;
|
||
PMMPFN Pfn2;
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
ASSERT ((PageFrameIndex != 0) && (PageFrameIndex <= MmHighestPhysicalPage) &&
|
||
(PageFrameIndex >= MmLowestPhysicalPage));
|
||
|
||
//
|
||
// Check to ensure the reference count for the page is zero.
|
||
//
|
||
|
||
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
|
||
|
||
MI_SNAP_DATA (Pfn1, Pfn1->PteAddress, 0xA);
|
||
|
||
ASSERT (Pfn1->u3.e1.CacheAttribute == MiCached);
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
|
||
MmModifiedNoWritePageListHead.Total += 1; // One more page on the list.
|
||
|
||
MI_TALLY_TRANSITION_PAGE_ADDITION (Pfn1);
|
||
|
||
first = MmModifiedNoWritePageListHead.Flink;
|
||
if (first == MM_EMPTY_LIST) {
|
||
|
||
//
|
||
// List is empty add the page to the ListHead.
|
||
//
|
||
|
||
MmModifiedNoWritePageListHead.Blink = PageFrameIndex;
|
||
}
|
||
else {
|
||
Pfn2 = MI_PFN_ELEMENT (first);
|
||
Pfn2->u2.Blink = PageFrameIndex;
|
||
}
|
||
|
||
MmModifiedNoWritePageListHead.Flink = PageFrameIndex;
|
||
Pfn1->u1.Flink = first;
|
||
Pfn1->u2.Blink = MM_EMPTY_LIST;
|
||
Pfn1->u3.e1.PageLocation = ModifiedNoWritePageList;
|
||
return;
|
||
}
|
||
|
||
PFN_NUMBER
|
||
MiAllocatePfn (
|
||
IN PMMPTE PointerPte,
|
||
IN ULONG Protection
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure allocates and initializes a page of memory.
|
||
|
||
Arguments:
|
||
|
||
PointerPte - Supplies the PTE to initialize.
|
||
|
||
Return Value:
|
||
|
||
The page frame index allocated.
|
||
|
||
Environment:
|
||
|
||
Kernel mode.
|
||
|
||
--*/
|
||
{
|
||
KIRQL OldIrql;
|
||
PFN_NUMBER PageFrameIndex;
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
MiEnsureAvailablePageOrWait (NULL, NULL);
|
||
|
||
PageFrameIndex = MiRemoveAnyPage (MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));
|
||
|
||
PointerPte->u.Long = MM_KERNEL_DEMAND_ZERO_PTE;
|
||
|
||
PointerPte->u.Soft.Protection |= Protection;
|
||
|
||
MiInitializePfn (PageFrameIndex, PointerPte, 1);
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
|
||
return PageFrameIndex;
|
||
}
|
||
|
||
VOID
|
||
FASTCALL
|
||
MiLogPfnInformation (
|
||
IN PMMPFN Pfn1,
|
||
IN USHORT Reason
|
||
)
|
||
{
|
||
MMPFN_IDENTITY PfnIdentity;
|
||
|
||
RtlZeroMemory (&PfnIdentity, sizeof(PfnIdentity));
|
||
|
||
if (Reason == PERFINFO_LOG_TYPE_INSERTINFREELIST) {
|
||
MI_MARK_PFN_UNDELETED (Pfn1);
|
||
}
|
||
|
||
MiIdentifyPfn(Pfn1, &PfnIdentity);
|
||
|
||
PerfInfoLogBytes (Reason, &PfnIdentity, sizeof(PfnIdentity));
|
||
|
||
if (Reason == PERFINFO_LOG_TYPE_INSERTINFREELIST) {
|
||
MI_SET_PFN_DELETED (Pfn1);
|
||
}
|
||
}
|
||
|
||
VOID
|
||
MiPurgeTransitionList (
|
||
VOID
|
||
)
|
||
{
|
||
PMMPFN Pfn1;
|
||
KIRQL OldIrql;
|
||
PFN_NUMBER PageFrameIndex;
|
||
|
||
//
|
||
// Run the transition list and free all the entries so transition
|
||
// faults are not satisfied for any of the non modified pages that were
|
||
// freed.
|
||
//
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
while (MmStandbyPageListHead.Total != 0) {
|
||
|
||
PageFrameIndex = MiRemovePageFromList (&MmStandbyPageListHead);
|
||
|
||
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
||
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
|
||
Pfn1->u3.e2.ReferenceCount += 1;
|
||
Pfn1->OriginalPte = ZeroPte;
|
||
|
||
MI_SET_PFN_DELETED (Pfn1);
|
||
|
||
MiDecrementReferenceCount (PageFrameIndex);
|
||
|
||
//
|
||
// If memory mirroring is in progress, any removal from
|
||
// the standby, modified or modified-no-write lists that isn't
|
||
// immediately re-inserting in one of these 3 lists must
|
||
// update the bitmap.
|
||
//
|
||
|
||
if (MiMirroringActive == TRUE) {
|
||
RtlSetBit (MiMirrorBitMap2, (ULONG)PageFrameIndex);
|
||
}
|
||
}
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
return;
|
||
}
|