able/windows-nt
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
windows-nt/Source/XPSP1/NT/base/ntos/mm/mi.h
CryptoAlgo Inc b3cac27891 Add source files
2020-09-26 16:20:57 +08:00

7484 lines
186 KiB
C
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*++
Copyright (c) 1989 Microsoft Corporation
Module Name:
mi.h
Abstract:
This module contains the private data structures and procedure
prototypes for the memory management system.
Author:
Lou Perazzoli (loup) 20-Mar-1989
Landy Wang (landyw) 02-Jun-1997
Revision History:
--*/
#ifndef _MI_
#define _MI_
#pragma warning(disable:4214) // bit field types other than int
#pragma warning(disable:4201) // nameless struct/union
#pragma warning(disable:4324) // alignment sensitive to declspec
#pragma warning(disable:4127) // condition expression is constant
#pragma warning(disable:4115) // named type definition in parentheses
#pragma warning(disable:4232) // dllimport not static
#pragma warning(disable:4206) // translation unit empty
#include "ntos.h"
#include "ntimage.h"
#include "ki.h"
#include "fsrtl.h"
#include "zwapi.h"
#include "pool.h"
#include "stdio.h"
#include "string.h"
#include "safeboot.h"
#include "triage.h"
#include "xip.h"
#if defined(_X86_)
#include "..\mm\i386\mi386.h"
#elif defined(_AMD64_)
#include "..\mm\amd64\miamd.h"
#elif defined(_IA64_)
#include "..\mm\ia64\miia64.h"
#else
#error "mm: a target architecture must be defined."
#endif
#if defined (_WIN64)
#define ASSERT32(exp)
#define ASSERT64(exp) ASSERT(exp)
#else
#define ASSERT32(exp) ASSERT(exp)
#define ASSERT64(exp)
#endif
//
// Special pool constants
//
#define MI_SPECIAL_POOL_PAGABLE 0x8000
#define MI_SPECIAL_POOL_VERIFIER 0x4000
#define MI_SPECIAL_POOL_IN_SESSION 0x2000
#define MI_SPECIAL_POOL_PTE_PAGABLE 0x0002
#define MI_SPECIAL_POOL_PTE_NONPAGABLE 0x0004
#define _2gb 0x80000000 // 2 gigabytes
#define _4gb 0x100000000 // 4 gigabytes
#define MM_FLUSH_COUNTER_MASK (0xFFFFF)
#define MM_FREE_WSLE_SHIFT 4
#define WSLE_NULL_INDEX ((((WSLE_NUMBER)-1) >> MM_FREE_WSLE_SHIFT))
#define MM_FREE_POOL_SIGNATURE (0x50554F4C)
#define MM_MINIMUM_PAGED_POOL_NTAS ((SIZE_T)(48*1024*1024))
#define MM_ALLOCATION_FILLS_VAD ((PMMPTE)(ULONG_PTR)~3)
#define MM_WORKING_SET_LIST_SEARCH 17
#define MM_FLUID_WORKING_SET 8
#define MM_FLUID_PHYSICAL_PAGES 32 //see MmResidentPages below.
#define MM_USABLE_PAGES_FREE 32
#define X64K (ULONG)65536
#define MM_HIGHEST_VAD_ADDRESS ((PVOID)((ULONG_PTR)MM_HIGHEST_USER_ADDRESS - (64 * 1024)))
#define MM_NO_WS_EXPANSION ((PLIST_ENTRY)0)
#define MM_WS_EXPANSION_IN_PROGRESS ((PLIST_ENTRY)35)
#define MM_WS_SWAPPED_OUT ((PLIST_ENTRY)37)
#define MM_IO_IN_PROGRESS ((PLIST_ENTRY)97) // MUST HAVE THE HIGHEST VALUE
#define MM4K_SHIFT 12 //MUST BE LESS THAN OR EQUAL TO PAGE_SHIFT
#define MM4K_MASK 0xfff
#define MMSECTOR_SHIFT 9 //MUST BE LESS THAN OR EQUAL TO PAGE_SHIFT
#define MMSECTOR_MASK 0x1ff
#define MM_LOCK_BY_REFCOUNT 0
#define MM_LOCK_BY_NONPAGE 1
#define MM_FORCE_TRIM 6
#define MM_GROW_WSLE_HASH 20
#define MM_MAXIMUM_WRITE_CLUSTER (MM_MAXIMUM_DISK_IO_SIZE / PAGE_SIZE)
//
// Number of PTEs to flush singularly before flushing the entire TB.
//
#define MM_MAXIMUM_FLUSH_COUNT (FLUSH_MULTIPLE_MAXIMUM-1)
//
// Page protections
//
#define MM_ZERO_ACCESS 0 // this value is not used.
#define MM_READONLY 1
#define MM_EXECUTE 2
#define MM_EXECUTE_READ 3
#define MM_READWRITE 4 // bit 2 is set if this is writable.
#define MM_WRITECOPY 5
#define MM_EXECUTE_READWRITE 6
#define MM_EXECUTE_WRITECOPY 7
#define MM_NOCACHE 0x8
#define MM_GUARD_PAGE 0x10
#define MM_DECOMMIT 0x10 //NO_ACCESS, Guard page
#define MM_NOACCESS 0x18 //NO_ACCESS, Guard_page, nocache.
#define MM_UNKNOWN_PROTECTION 0x100 //bigger than 5 bits!
#define MM_LARGE_PAGES 0x111
#define MM_INVALID_PROTECTION ((ULONG)-1) //bigger than 5 bits!
#define MM_KSTACK_OUTSWAPPED 0x1F // Denotes outswapped kernel stack pages.
#define MM_PROTECTION_WRITE_MASK 4
#define MM_PROTECTION_COPY_MASK 1
#define MM_PROTECTION_OPERATION_MASK 7 // mask off guard page and nocache.
#define MM_PROTECTION_EXECUTE_MASK 2
#define MM_SECURE_DELETE_CHECK 0x55
//
// Debug flags
//
#define MM_DBG_WRITEFAULT 0x1
#define MM_DBG_PTE_UPDATE 0x2
#define MM_DBG_DUMP_WSL 0x4
#define MM_DBG_PAGEFAULT 0x8
#define MM_DBG_WS_EXPANSION 0x10
#define MM_DBG_MOD_WRITE 0x20
#define MM_DBG_CHECK_PTE 0x40
#define MM_DBG_VAD_CONFLICT 0x80
#define MM_DBG_SECTIONS 0x100
#define MM_DBG_SYS_PTES 0x400
#define MM_DBG_CLEAN_PROCESS 0x800
#define MM_DBG_COLLIDED_PAGE 0x1000
#define MM_DBG_DUMP_BOOT_PTES 0x2000
#define MM_DBG_FORK 0x4000
#define MM_DBG_DIR_BASE 0x8000
#define MM_DBG_FLUSH_SECTION 0x10000
#define MM_DBG_PRINTS_MODWRITES 0x20000
#define MM_DBG_PAGE_IN_LIST 0x40000
#define MM_DBG_CHECK_PFN_LOCK 0x80000
#define MM_DBG_PRIVATE_PAGES 0x100000
#define MM_DBG_WALK_VAD_TREE 0x200000
#define MM_DBG_SWAP_PROCESS 0x400000
#define MM_DBG_LOCK_CODE 0x800000
#define MM_DBG_STOP_ON_ACCVIO 0x1000000
#define MM_DBG_PAGE_REF_COUNT 0x2000000
#define MM_DBG_SHOW_FAULTS 0x40000000
#define MM_DBG_SESSIONS 0x80000000
//
// If the PTE.protection & MM_COPY_ON_WRITE_MASK == MM_COPY_ON_WRITE_MASK
// then the PTE is copy on write.
//
#define MM_COPY_ON_WRITE_MASK 5
extern ULONG MmProtectToValue[32];
extern
#if (defined(_WIN64) || defined(_X86PAE_))
ULONGLONG
#else
ULONG
#endif
MmProtectToPteMask[32];
extern ULONG MmMakeProtectNotWriteCopy[32];
extern ACCESS_MASK MmMakeSectionAccess[8];
extern ACCESS_MASK MmMakeFileAccess[8];
//
// Time constants
//
extern const LARGE_INTEGER MmSevenMinutes;
extern LARGE_INTEGER MmWorkingSetProtectionTime;
const extern LARGE_INTEGER MmOneSecond;
const extern LARGE_INTEGER MmTwentySeconds;
const extern LARGE_INTEGER MmShortTime;
const extern LARGE_INTEGER MmHalfSecond;
const extern LARGE_INTEGER Mm30Milliseconds;
extern LARGE_INTEGER MmCriticalSectionTimeout;
//
// A month's worth
//
extern ULONG MmCritsectTimeoutSeconds;
//
// this is the csrss process !
//
extern PEPROCESS ExpDefaultErrorPortProcess;
extern SIZE_T MmExtendedCommit;
extern SIZE_T MmTotalProcessCommit;
//
// The total number of pages needed for the loader to successfully hibernate.
//
extern PFN_NUMBER MmHiberPages;
//
// The counters and reasons to retry IO to protect against verifier induced
// failures and temporary conditions.
//
extern ULONG MiIoRetryMask;
extern ULONG MiFaultRetryMask;
extern ULONG MiUserFaultRetryMask;
#define MmIsRetryIoStatus(S) (((S) == STATUS_INSUFFICIENT_RESOURCES) || \
((S) == STATUS_WORKING_SET_QUOTA) || \
((S) == STATUS_NO_MEMORY))
#if defined (_MI_MORE_THAN_4GB_)
extern PFN_NUMBER MiNoLowMemory;
#if defined (_WIN64)
#define MI_MAGIC_4GB_RECLAIM 0xffffedf0
#else
#define MI_MAGIC_4GB_RECLAIM 0xffedf0
#endif
#define MI_LOWMEM_MAGIC_BIT (0x80000000)
extern PRTL_BITMAP MiLowMemoryBitMap;
#endif
//
// This is a version of COMPUTE_PAGES_SPANNED that works for 32 and 64 ranges.
//
#define MI_COMPUTE_PAGES_SPANNED(Va, Size) \
((((ULONG_PTR)(Va) & (PAGE_SIZE -1)) + (Size) + (PAGE_SIZE - 1)) >> PAGE_SHIFT)
//++
//
// ULONG
// MI_CONVERT_FROM_PTE_PROTECTION (
// IN ULONG PROTECTION_MASK
// )
//
// Routine Description:
//
// This routine converts a PTE protection into a Protect value.
//
// Arguments:
//
//
// Return Value:
//
// Returns the
//
//--
#define MI_CONVERT_FROM_PTE_PROTECTION(PROTECTION_MASK) \
(MmProtectToValue[PROTECTION_MASK])
#define MI_MASK_TO_PTE(PROTECTION_MASK) MmProtectToPteMask[PROTECTION_MASK]
#define MI_IS_PTE_PROTECTION_COPY_WRITE(PROTECTION_MASK) \
(((PROTECTION_MASK) & MM_COPY_ON_WRITE_MASK) == MM_COPY_ON_WRITE_MASK)
//++
//
// ULONG
// MI_ROUND_TO_64K (
// IN ULONG LENGTH
// )
//
// Routine Description:
//
//
// The ROUND_TO_64k macro takes a LENGTH in bytes and rounds it up to a multiple
// of 64K.
//
// Arguments:
//
// LENGTH - LENGTH in bytes to round up to 64k.
//
// Return Value:
//
// Returns the LENGTH rounded up to a multiple of 64k.
//
//--
#define MI_ROUND_TO_64K(LENGTH) (((LENGTH) + X64K - 1) & ~((ULONG_PTR)X64K - 1))
extern ULONG MiLastVadBit;
//++
//
// ULONG
// MI_ROUND_TO_SIZE (
// IN ULONG LENGTH,
// IN ULONG ALIGNMENT
// )
//
// Routine Description:
//
//
// The ROUND_TO_SIZE macro takes a LENGTH in bytes and rounds it up to a
// multiple of the alignment.
//
// Arguments:
//
// LENGTH - LENGTH in bytes to round up to.
//
// ALIGNMENT - alignment to round to, must be a power of 2, e.g, 2**n.
//
// Return Value:
//
// Returns the LENGTH rounded up to a multiple of the alignment.
//
//--
#define MI_ROUND_TO_SIZE(LENGTH,ALIGNMENT) \
(((LENGTH) + ((ALIGNMENT) - 1)) & ~((ALIGNMENT) - 1))
//++
//
// PVOID
// MI_64K_ALIGN (
// IN PVOID VA
// )
//
// Routine Description:
//
//
// The MI_64K_ALIGN macro takes a virtual address and returns a 64k-aligned
// virtual address for that page.
//
// Arguments:
//
// VA - Virtual address.
//
// Return Value:
//
// Returns the 64k aligned virtual address.
//
//--
#define MI_64K_ALIGN(VA) ((PVOID)((ULONG_PTR)(VA) & ~((LONG)X64K - 1)))
//++
//
// PVOID
// MI_ALIGN_TO_SIZE (
// IN PVOID VA
// IN ULONG ALIGNMENT
// )
//
// Routine Description:
//
//
// The MI_ALIGN_TO_SIZE macro takes a virtual address and returns a
// virtual address for that page with the specified alignment.
//
// Arguments:
//
// VA - Virtual address.
//
// ALIGNMENT - alignment to round to, must be a power of 2, e.g, 2**n.
//
// Return Value:
//
// Returns the aligned virtual address.
//
//--
#define MI_ALIGN_TO_SIZE(VA,ALIGNMENT) ((PVOID)((ULONG_PTR)(VA) & ~((ULONG_PTR) ALIGNMENT - 1)))
//++
//
// LONGLONG
// MI_STARTING_OFFSET (
// IN PSUBSECTION SUBSECT
// IN PMMPTE PTE
// )
//
// Routine Description:
//
// This macro takes a pointer to a PTE within a subsection and a pointer
// to that subsection and calculates the offset for that PTE within the
// file.
//
// Arguments:
//
// PTE - PTE within subsection.
//
// SUBSECT - Subsection
//
// Return Value:
//
// Offset for issuing I/O from.
//
//--
#define MI_STARTING_OFFSET(SUBSECT,PTE) \
(((LONGLONG)((ULONG_PTR)((PTE) - ((SUBSECT)->SubsectionBase))) << PAGE_SHIFT) + \
((LONGLONG)((SUBSECT)->StartingSector) << MMSECTOR_SHIFT));
// NTSTATUS
// MiFindEmptyAddressRangeDown (
// IN ULONG_PTR SizeOfRange,
// IN PVOID HighestAddressToEndAt,
// IN ULONG_PTR Alignment,
// OUT PVOID *Base
// )
//
//
// Routine Description:
//
// The function examines the virtual address descriptors to locate
// an unused range of the specified size and returns the starting
// address of the range. This routine looks from the top down.
//
// Arguments:
//
// SizeOfRange - Supplies the size in bytes of the range to locate.
//
// HighestAddressToEndAt - Supplies the virtual address to begin looking
// at.
//
// Alignment - Supplies the alignment for the address. Must be
// a power of 2 and greater than the page_size.
//
//Return Value:
//
// Returns the starting address of a suitable range.
//
#define MiFindEmptyAddressRangeDown(Root,SizeOfRange,HighestAddressToEndAt,Alignment,Base) \
(MiFindEmptyAddressRangeDownTree( \
(SizeOfRange), \
(HighestAddressToEndAt), \
(Alignment), \
(PMMADDRESS_NODE)(Root), \
(Base)))
// PMMVAD
// MiGetPreviousVad (
// IN PMMVAD Vad
// )
//
// Routine Description:
//
// This function locates the virtual address descriptor which contains
// the address range which logically precedes the specified virtual
// address descriptor.
//
// Arguments:
//
// Vad - Supplies a pointer to a virtual address descriptor.
//
// Return Value:
//
// Returns a pointer to the virtual address descriptor containing the
// next address range, NULL if none.
//
//
#define MiGetPreviousVad(VAD) ((PMMVAD)MiGetPreviousNode((PMMADDRESS_NODE)(VAD)))
// PMMVAD
// MiGetNextVad (
// IN PMMVAD Vad
// )
//
// Routine Description:
//
// This function locates the virtual address descriptor which contains
// the address range which logically follows the specified address range.
//
// Arguments:
//
// VAD - Supplies a pointer to a virtual address descriptor.
//
// Return Value:
//
// Returns a pointer to the virtual address descriptor containing the
// next address range, NULL if none.
//
#define MiGetNextVad(VAD) ((PMMVAD)MiGetNextNode((PMMADDRESS_NODE)(VAD)))
// PMMVAD
// MiGetFirstVad (
// Process
// )
//
// Routine Description:
//
// This function locates the virtual address descriptor which contains
// the address range which logically is first within the address space.
//
// Arguments:
//
// Process - Specifies the process in which to locate the VAD.
//
// Return Value:
//
// Returns a pointer to the virtual address descriptor containing the
// first address range, NULL if none.
#define MiGetFirstVad(Process) \
((PMMVAD)MiGetFirstNode((PMMADDRESS_NODE)(Process->VadRoot)))
LOGICAL
MiCheckForConflictingVadExistence (
IN PEPROCESS Process,
IN PVOID StartingAddress,
IN PVOID EndingAddress
);
// PMMVAD
// MiCheckForConflictingVad (
// IN PVOID StartingAddress,
// IN PVOID EndingAddress
// )
//
// Routine Description:
//
// The function determines if any addresses between a given starting and
// ending address is contained within a virtual address descriptor.
//
// Arguments:
//
// StartingAddress - Supplies the virtual address to locate a containing
// descriptor.
//
// EndingAddress - Supplies the virtual address to locate a containing
// descriptor.
//
// Return Value:
//
// Returns a pointer to the first conflicting virtual address descriptor
// if one is found, otherwise a NULL value is returned.
//
#define MiCheckForConflictingVad(CurrentProcess,StartingAddress,EndingAddress) \
((PMMVAD)MiCheckForConflictingNode( \
MI_VA_TO_VPN(StartingAddress), \
MI_VA_TO_VPN(EndingAddress), \
(PMMADDRESS_NODE)(CurrentProcess->VadRoot)))
// PMMCLONE_DESCRIPTOR
// MiGetNextClone (
// IN PMMCLONE_DESCRIPTOR Clone
// )
//
// Routine Description:
//
// This function locates the virtual address descriptor which contains
// the address range which logically follows the specified address range.
//
// Arguments:
//
// Clone - Supplies a pointer to a virtual address descriptor.
//
// Return Value:
//
// Returns a pointer to the virtual address descriptor containing the
// next address range, NULL if none.
//
//
#define MiGetNextClone(CLONE) \
((PMMCLONE_DESCRIPTOR)MiGetNextNode((PMMADDRESS_NODE)(CLONE)))
// PMMCLONE_DESCRIPTOR
// MiGetPreviousClone (
// IN PMMCLONE_DESCRIPTOR Clone
// )
//
// Routine Description:
//
// This function locates the virtual address descriptor which contains
// the address range which logically precedes the specified virtual
// address descriptor.
//
// Arguments:
//
// Clone - Supplies a pointer to a virtual address descriptor.
//
// Return Value:
//
// Returns a pointer to the virtual address descriptor containing the
// next address range, NULL if none.
#define MiGetPreviousClone(CLONE) \
((PMMCLONE_DESCRIPTOR)MiGetPreviousNode((PMMADDRESS_NODE)(CLONE)))
// PMMCLONE_DESCRIPTOR
// MiGetFirstClone (
// )
//
// Routine Description:
//
// This function locates the virtual address descriptor which contains
// the address range which logically is first within the address space.
//
// Arguments:
//
// None.
//
// Return Value:
//
// Returns a pointer to the virtual address descriptor containing the
// first address range, NULL if none.
//
#define MiGetFirstClone(_CurrentProcess) \
((PMMCLONE_DESCRIPTOR)MiGetFirstNode((PMMADDRESS_NODE)(_CurrentProcess->CloneRoot)))
// VOID
// MiInsertClone (
// IN PMMCLONE_DESCRIPTOR Clone
// )
//
// Routine Description:
//
// This function inserts a virtual address descriptor into the tree and
// reorders the splay tree as appropriate.
//
// Arguments:
//
// Clone - Supplies a pointer to a virtual address descriptor
//
//
// Return Value:
//
// None.
//
#define MiInsertClone(_CurrentProcess, CLONE) \
{ \
ASSERT ((CLONE)->NumberOfPtes != 0); \
MiInsertNode(((PMMADDRESS_NODE)(CLONE)),(PMMADDRESS_NODE *)&(_CurrentProcess->CloneRoot)); \
}
// VOID
// MiRemoveClone (
// IN PMMCLONE_DESCRIPTOR Clone
// )
//
// Routine Description:
//
// This function removes a virtual address descriptor from the tree and
// reorders the splay tree as appropriate.
//
// Arguments:
//
// Clone - Supplies a pointer to a virtual address descriptor.
//
// Return Value:
//
// None.
//
#define MiRemoveClone(_CurrentProcess, CLONE) \
MiRemoveNode((PMMADDRESS_NODE)(CLONE),(PMMADDRESS_NODE *)&(_CurrentProcess->CloneRoot));
// PMMCLONE_DESCRIPTOR
// MiLocateCloneAddress (
// IN PVOID VirtualAddress
// )
//
// /*++
//
// Routine Description:
//
// The function locates the virtual address descriptor which describes
// a given address.
//
// Arguments:
//
// VirtualAddress - Supplies the virtual address to locate a descriptor
// for.
//
// Return Value:
//
// Returns a pointer to the virtual address descriptor which contains
// the supplied virtual address or NULL if none was located.
//
#define MiLocateCloneAddress(_CurrentProcess, VA) \
(_CurrentProcess->CloneRoot ? \
((PMMCLONE_DESCRIPTOR)MiLocateAddressInTree(((ULONG_PTR)VA), \
(PMMADDRESS_NODE *)&(_CurrentProcess->CloneRoot))) : \
NULL)
#define MI_VA_TO_PAGE(va) ((ULONG_PTR)(va) >> PAGE_SHIFT)
#define MI_VA_TO_VPN(va) ((ULONG_PTR)(va) >> PAGE_SHIFT)
#define MI_VPN_TO_VA(vpn) (PVOID)((vpn) << PAGE_SHIFT)
#define MI_VPN_TO_VA_ENDING(vpn) (PVOID)(((vpn) << PAGE_SHIFT) | (PAGE_SIZE - 1))
#define MiGetByteOffset(va) ((ULONG_PTR)(va) & (PAGE_SIZE - 1))
#define MI_PFN_ELEMENT(index) (&MmPfnDatabase[index])
//
// Make a write-copy PTE, only writable.
//
#define MI_MAKE_PROTECT_NOT_WRITE_COPY(PROTECT) \
(MmMakeProtectNotWriteCopy[PROTECT])
//
// Define macros to lock and unlock the PFN database.
//
#define MiLockPfnDatabase(OldIrql) \
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock)
#define MiUnlockPfnDatabase(OldIrql) \
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql)
#define MiLockPfnDatabaseAtDpcLevel() \
KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock])
#define MiUnlockPfnDatabaseFromDpcLevel() \
KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock])
#define MiTryToLockPfnDatabase(OldIrql) \
KeTryToAcquireQueuedSpinLock(LockQueuePfnLock, &OldIrql)
#define MiReleasePfnLock() \
KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock])
#define MiLockSystemSpace(OldIrql) \
OldIrql = KeAcquireQueuedSpinLock(LockQueueSystemSpaceLock)
#define MiUnlockSystemSpace(OldIrql) \
KeReleaseQueuedSpinLock(LockQueueSystemSpaceLock, OldIrql)
#define MiLockSystemSpaceAtDpcLevel() \
KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueueSystemSpaceLock])
#define MiUnlockSystemSpaceFromDpcLevel() \
KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->LockQueue[LockQueueSystemSpaceLock])
// #define _MI_INSTRUMENT_PFN 1
#if defined (_MI_INSTRUMENT_PFN)
#define MI_MAX_PFN_CALLERS 500
typedef struct _MMPFNTIMINGS {
LARGE_INTEGER HoldTime; // Low bit is set if another processor waited
ULONG_PTR AcquiredAddress;
ULONG_PTR ReleasedAddress;
} MMPFNTIMINGS, *PMMPFNTIMINGS;
extern ULONG MiPfnTimings;
extern ULONG_PTR MiPfnAcquiredAddress;
extern MMPFNTIMINGS MiPfnSorted[];
extern LARGE_INTEGER MiPfnAcquired;
extern LARGE_INTEGER MiPfnThreshold;
ULONG_PTR
MiGetExecutionAddress(
VOID
);
#if defined(_X86_)
#define MI_GET_EXECUTION_ADDRESS(varname) varname = MiGetExecutionAddress();
#else
#define MI_GET_EXECUTION_ADDRESS(varname) varname = 0;
#endif
#define LOCK_PFN_TIMESTAMP() \
{ \
MiPfnAcquired = KeQueryPerformanceCounter (NULL);\
MI_GET_EXECUTION_ADDRESS(MiPfnAcquiredAddress); \
}
#define UNLOCK_PFN_TIMESTAMP() \
{ \
ULONG i; \
ULONG_PTR ExecutionAddress; \
LARGE_INTEGER PfnReleased; \
LARGE_INTEGER PfnHoldTime; \
PfnReleased = KeQueryPerformanceCounter (NULL); \
MI_GET_EXECUTION_ADDRESS(ExecutionAddress); \
PfnHoldTime.QuadPart = (PfnReleased.QuadPart - MiPfnAcquired.QuadPart) & ~0x1; \
i = MI_MAX_PFN_CALLERS - 1; \
do { \
if (PfnHoldTime.QuadPart < MiPfnSorted[i].HoldTime.QuadPart) { \
break; \
} \
i -= 1; \
} while (i != (ULONG)-1); \
if (i != MI_MAX_PFN_CALLERS - 1) { \
i += 1; \
if (i != MI_MAX_PFN_CALLERS - 1) { \
RtlMoveMemory (&MiPfnSorted[i+1], &MiPfnSorted[i], (MI_MAX_PFN_CALLERS-(i+1)) * sizeof(MMPFNTIMINGS)); \
} \
MiPfnSorted[i].HoldTime = PfnHoldTime; \
if (KeTestForWaitersQueuedSpinLock(LockQueuePfnLock) == TRUE) {\
MiPfnSorted[i].HoldTime.LowPart |= 0x1; \
} \
MiPfnSorted[i].AcquiredAddress = MiPfnAcquiredAddress; \
MiPfnSorted[i].ReleasedAddress = ExecutionAddress; \
} \
if ((MiPfnTimings & 0x2) && (PfnHoldTime.QuadPart >= MiPfnThreshold.QuadPart)) { \
DbgBreakPoint (); \
} \
if (MiPfnTimings & 0x1) { \
MiPfnTimings &= ~0x1; \
RtlZeroMemory (&MiPfnSorted[0], MI_MAX_PFN_CALLERS * sizeof(MMPFNTIMINGS)); \
} \
}
#else
#define LOCK_PFN_TIMESTAMP()
#define UNLOCK_PFN_TIMESTAMP()
#endif
#define LOCK_PFN(OLDIRQL) ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
MiLockPfnDatabase(OLDIRQL); \
LOCK_PFN_TIMESTAMP();
#define LOCK_PFN_WITH_TRY(OLDIRQL) \
ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
do { \
} while (MiTryToLockPfnDatabase(OLDIRQL) == FALSE); \
LOCK_PFN_TIMESTAMP();
#define UNLOCK_PFN(OLDIRQL) \
UNLOCK_PFN_TIMESTAMP(); \
MiUnlockPfnDatabase(OLDIRQL); \
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
#define LOCK_PFN2(OLDIRQL) ASSERT (KeGetCurrentIrql() <= DISPATCH_LEVEL); \
MiLockPfnDatabase(OLDIRQL); \
LOCK_PFN_TIMESTAMP();
#define UNLOCK_PFN2(OLDIRQL) \
UNLOCK_PFN_TIMESTAMP(); \
MiUnlockPfnDatabase(OLDIRQL); \
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
#define LOCK_PFN_AT_DPC() ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
MiLockPfnDatabaseAtDpcLevel(); \
LOCK_PFN_TIMESTAMP();
#define UNLOCK_PFN_FROM_DPC() \
UNLOCK_PFN_TIMESTAMP(); \
MiUnlockPfnDatabaseFromDpcLevel(); \
ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
#define UNLOCK_PFN_AND_THEN_WAIT(OLDIRQL) \
{ \
KIRQL XXX; \
ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
ASSERT (OLDIRQL <= APC_LEVEL); \
KiLockDispatcherDatabase (&XXX); \
UNLOCK_PFN_TIMESTAMP(); \
MiReleasePfnLock(); \
(KeGetCurrentThread())->WaitIrql = OLDIRQL; \
(KeGetCurrentThread())->WaitNext = TRUE; \
}
extern KMUTANT MmSystemLoadLock;
#if DBG
#define SYSLOAD_LOCK_OWNED_BY_ME() ((PETHREAD)MmSystemLoadLock.OwnerThread == PsGetCurrentThread())
#else
#define SYSLOAD_LOCK_OWNED_BY_ME()
#endif
#if DBG
#if defined (_MI_COMPRESSION)
extern KIRQL MiCompressionIrql;
#define MM_PFN_LOCK_ASSERT() \
if (MmDebug & 0x80000) { \
KIRQL _OldIrql; \
_OldIrql = KeGetCurrentIrql(); \
ASSERT ((_OldIrql == DISPATCH_LEVEL) || \
((MiCompressionIrql != 0) && (_OldIrql == MiCompressionIrql))); \
}
#else
#define MM_PFN_LOCK_ASSERT() \
if (MmDebug & 0x80000) { \
KIRQL _OldIrql; \
_OldIrql = KeGetCurrentIrql(); \
ASSERT (_OldIrql == DISPATCH_LEVEL); \
}
#endif
extern PETHREAD MiExpansionLockOwner;
#define MM_SET_EXPANSION_OWNER() ASSERT (MiExpansionLockOwner == NULL); \
MiExpansionLockOwner = PsGetCurrentThread();
#define MM_CLEAR_EXPANSION_OWNER() ASSERT (MiExpansionLockOwner == PsGetCurrentThread()); \
MiExpansionLockOwner = NULL;
#else
#define MM_PFN_LOCK_ASSERT()
#define MM_SET_EXPANSION_OWNER()
#define MM_CLEAR_EXPANSION_OWNER()
#endif //DBG
#define LOCK_EXPANSION(OLDIRQL) ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
ExAcquireSpinLock (&MmExpansionLock, &OLDIRQL);\
MM_SET_EXPANSION_OWNER ();
#define UNLOCK_EXPANSION(OLDIRQL) MM_CLEAR_EXPANSION_OWNER (); \
ExReleaseSpinLock (&MmExpansionLock, OLDIRQL); \
ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
#define UNLOCK_EXPANSION_AND_THEN_WAIT(OLDIRQL) \
{ \
KIRQL XXX; \
ASSERT (KeGetCurrentIrql() == 2); \
ASSERT (OLDIRQL <= APC_LEVEL); \
KiLockDispatcherDatabase (&XXX); \
MM_CLEAR_EXPANSION_OWNER (); \
KiReleaseSpinLock (&MmExpansionLock); \
(KeGetCurrentThread())->WaitIrql = OLDIRQL; \
(KeGetCurrentThread())->WaitNext = TRUE; \
}
extern PETHREAD MmSystemLockOwner;
#define LOCK_SYSTEM_WS(OLDIRQL,_Thread) \
ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
KeRaiseIrql(APC_LEVEL,&OLDIRQL); \
ExAcquireResourceExclusiveLite(&MmSystemWsLock,TRUE); \
ASSERT (MmSystemLockOwner == NULL); \
MmSystemLockOwner = _Thread;
#define LOCK_SYSTEM_WS_UNSAFE(_Thread) \
ASSERT (KeGetCurrentIrql() == APC_LEVEL); \
ExAcquireResourceExclusiveLite(&MmSystemWsLock,TRUE); \
ASSERT (MmSystemLockOwner == NULL); \
MmSystemLockOwner = _Thread;
#define UNLOCK_SYSTEM_WS(OLDIRQL) \
ASSERT (MmSystemLockOwner == PsGetCurrentThread()); \
MmSystemLockOwner = NULL; \
ExReleaseResourceLite (&MmSystemWsLock); \
KeLowerIrql (OLDIRQL); \
ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
#define UNLOCK_SYSTEM_WS_NO_IRQL() \
ASSERT (MmSystemLockOwner == PsGetCurrentThread()); \
MmSystemLockOwner = NULL; \
ExReleaseResourceLite (&MmSystemWsLock);
#define MM_SYSTEM_WS_LOCK_ASSERT() \
ASSERT (PsGetCurrentThread() == MmSystemLockOwner);
#define LOCK_HYPERSPACE(_Process, OLDIRQL) \
ASSERT (_Process == PsGetCurrentProcess ()); \
ExAcquireSpinLock (&_Process->HyperSpaceLock, OLDIRQL);
#define UNLOCK_HYPERSPACE(_Process, VA, OLDIRQL) \
ASSERT (_Process == PsGetCurrentProcess ()); \
MiGetPteAddress(VA)->u.Long = 0; \
ExReleaseSpinLock (&_Process->HyperSpaceLock, OLDIRQL);
#define LOCK_HYPERSPACE_AT_DPC(_Process) \
ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
ASSERT (_Process == PsGetCurrentProcess ()); \
ExAcquireSpinLockAtDpcLevel (&_Process->HyperSpaceLock);
#define UNLOCK_HYPERSPACE_FROM_DPC(_Process, VA) \
ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL); \
ASSERT (_Process == PsGetCurrentProcess ()); \
MiGetPteAddress(VA)->u.Long = 0; \
ExReleaseSpinLockFromDpcLevel (&_Process->HyperSpaceLock);
#define MiUnmapPageInHyperSpace(_Process, VA, OLDIRQL) UNLOCK_HYPERSPACE(_Process, VA, OLDIRQL)
#define MiUnmapPageInHyperSpaceFromDpc(_Process, VA) UNLOCK_HYPERSPACE_FROM_DPC(_Process, VA)
#if defined (_AXP64_)
#define MI_WS_OWNER(PROCESS) ((PROCESS)->WorkingSetLock.Owner == KeGetCurrentThread())
#define MI_NOT_WS_OWNER(PROCESS) (!MI_WS_OWNER(PROCESS))
#else
#define MI_WS_OWNER(PROCESS) 1
#define MI_NOT_WS_OWNER(PROCESS) 1
#endif
#define MI_MUTEX_ACQUIRED_UNSAFE 0x88
#define MI_IS_WS_UNSAFE(PROCESS) ((PROCESS)->WorkingSetAcquiredUnsafe == MI_MUTEX_ACQUIRED_UNSAFE)
#define LOCK_WS(PROCESS) \
ASSERT (MI_NOT_WS_OWNER(PROCESS)); \
ExAcquireFastMutex( &((PROCESS)->WorkingSetLock)); \
ASSERT (!MI_IS_WS_UNSAFE(PROCESS));
#define LOCK_WS_UNSAFE(PROCESS) \
ASSERT (MI_NOT_WS_OWNER(PROCESS)); \
ASSERT (KeGetCurrentIrql() == APC_LEVEL); \
ExAcquireFastMutexUnsafe( &((PROCESS)->WorkingSetLock));\
(PROCESS)->WorkingSetAcquiredUnsafe = MI_MUTEX_ACQUIRED_UNSAFE;
#define MI_MUST_BE_UNSAFE(PROCESS) \
ASSERT (KeGetCurrentIrql() == APC_LEVEL); \
ASSERT (MI_WS_OWNER(PROCESS)); \
ASSERT (MI_IS_WS_UNSAFE(PROCESS));
#define MI_MUST_BE_SAFE(PROCESS) \
ASSERT (MI_WS_OWNER(PROCESS)); \
ASSERT (!MI_IS_WS_UNSAFE(PROCESS));
#if 0
#define MI_MUST_BE_UNSAFE(PROCESS) \
if (KeGetCurrentIrql() != APC_LEVEL) { \
KeBugCheckEx(MEMORY_MANAGEMENT, 0x32, (ULONG_PTR)PROCESS, KeGetCurrentIrql(), 0); \
} \
if (!MI_WS_OWNER(PROCESS)) { \
KeBugCheckEx(MEMORY_MANAGEMENT, 0x33, (ULONG_PTR)PROCESS, 0, 0); \
} \
if (!MI_IS_WS_UNSAFE(PROCESS)) { \
KeBugCheckEx(MEMORY_MANAGEMENT, 0x34, (ULONG_PTR)PROCESS, 0, 0); \
}
#define MI_MUST_BE_SAFE(PROCESS) \
if (!MI_WS_OWNER(PROCESS)) { \
KeBugCheckEx(MEMORY_MANAGEMENT, 0x42, (ULONG_PTR)PROCESS, 0, 0); \
} \
if (MI_IS_WS_UNSAFE(PROCESS)) { \
KeBugCheckEx(MEMORY_MANAGEMENT, 0x43, (ULONG_PTR)PROCESS, 0, 0); \
}
#endif
#define UNLOCK_WS(PROCESS) \
MI_MUST_BE_SAFE(PROCESS); \
ExReleaseFastMutex(&((PROCESS)->WorkingSetLock));
#define UNLOCK_WS_UNSAFE(PROCESS) \
MI_MUST_BE_UNSAFE(PROCESS); \
(PROCESS)->WorkingSetAcquiredUnsafe = 0; \
ExReleaseFastMutexUnsafe(&((PROCESS)->WorkingSetLock)); \
ASSERT (KeGetCurrentIrql() == APC_LEVEL);
#define LOCK_ADDRESS_SPACE(PROCESS) \
ExAcquireFastMutex( &((PROCESS)->AddressCreationLock))
#define LOCK_WS_AND_ADDRESS_SPACE(PROCESS) \
LOCK_ADDRESS_SPACE(PROCESS); \
LOCK_WS_UNSAFE(PROCESS);
#define UNLOCK_WS_AND_ADDRESS_SPACE(PROCESS) \
UNLOCK_WS_UNSAFE(PROCESS); \
UNLOCK_ADDRESS_SPACE(PROCESS);
#define UNLOCK_ADDRESS_SPACE(PROCESS) \
ExReleaseFastMutex( &((PROCESS)->AddressCreationLock))
//
// The working set lock may have been acquired safely or unsafely.
// Release and reacquire it regardless.
//
#define UNLOCK_WS_REGARDLESS(PROCESS, WSHELDSAFE) \
ASSERT (MI_WS_OWNER (PROCESS)); \
if (MI_IS_WS_UNSAFE (PROCESS)) { \
UNLOCK_WS_UNSAFE (PROCESS); \
WSHELDSAFE = FALSE; \
} \
else { \
UNLOCK_WS (PROCESS); \
WSHELDSAFE = TRUE; \
}
#define LOCK_WS_REGARDLESS(PROCESS, WSHELDSAFE) \
if (WSHELDSAFE == TRUE) { \
LOCK_WS (PROCESS); \
} \
else { \
LOCK_WS_UNSAFE (PROCESS); \
}
#define ZERO_LARGE(LargeInteger) \
(LargeInteger).LowPart = 0; \
(LargeInteger).HighPart = 0;
#define NO_BITS_FOUND ((ULONG)-1)
//++
//
// ULONG
// MI_CHECK_BIT (
// IN PULONG ARRAY
// IN ULONG BIT
// )
//
// Routine Description:
//
// The MI_CHECK_BIT macro checks to see if the specified bit is
// set within the specified array.
//
// Arguments:
//
// ARRAY - First element of the array to check.
//
// BIT - bit number (first bit is 0) to check.
//
// Return Value:
//
// Returns the value of the bit (0 or 1).
//
//--
#define MI_CHECK_BIT(ARRAY,BIT) \
(((ULONG)ARRAY[(BIT) / (sizeof(ULONG)*8)] >> ((BIT) & 0x1F)) & 1)
//++
//
// VOID
// MI_SET_BIT (
// IN PULONG ARRAY
// IN ULONG BIT
// )
//
// Routine Description:
//
// The MI_SET_BIT macro sets the specified bit within the
// specified array.
//
// Arguments:
//
// ARRAY - First element of the array to set.
//
// BIT - bit number.
//
// Return Value:
//
// None.
//
//--
#define MI_SET_BIT(ARRAY,BIT) \
ARRAY[(BIT) / (sizeof(ULONG)*8)] |= (1 << ((BIT) & 0x1F))
//++
//
// VOID
// MI_CLEAR_BIT (
// IN PULONG ARRAY
// IN ULONG BIT
// )
//
// Routine Description:
//
// The MI_CLEAR_BIT macro sets the specified bit within the
// specified array.
//
// Arguments:
//
// ARRAY - First element of the array to clear.
//
// BIT - bit number.
//
// Return Value:
//
// None.
//
//--
#define MI_CLEAR_BIT(ARRAY,BIT) \
ARRAY[(BIT) / (sizeof(ULONG)*8)] &= ~(1 << ((BIT) & 0x1F))
//
// These control the mirroring capabilities.
//
extern ULONG MmMirroring;
#define MM_MIRRORING_ENABLED 0x1 // Enable mirroring capabilities.
#define MM_MIRRORING_VERIFYING 0x2 // The HAL wants to verify the copy.
extern PRTL_BITMAP MiMirrorBitMap;
extern PRTL_BITMAP MiMirrorBitMap2;
extern LOGICAL MiMirroringActive;
#if defined (_WIN64)
#define MI_MAGIC_AWE_PTEFRAME 0xffffedcb
#else
#define MI_MAGIC_AWE_PTEFRAME 0xffedcb
#endif
#define MI_PFN_IS_AWE(Pfn1) \
((Pfn1->u2.ShareCount <= 3) && \
(Pfn1->u3.e1.PageLocation == ActiveAndValid) && \
(Pfn1->u4.VerifierAllocation == 0) && \
(Pfn1->u3.e1.LargeSessionAllocation == 0) && \
(Pfn1->u3.e1.StartOfAllocation == 1) && \
(Pfn1->u3.e1.EndOfAllocation == 1) && \
(Pfn1->u4.PteFrame == MI_MAGIC_AWE_PTEFRAME))
//
// PFN database element.
//
//
// Define pseudo fields for start and end of allocation.
//
#define StartOfAllocation ReadInProgress
#define EndOfAllocation WriteInProgress
#define LargeSessionAllocation PrototypePte
typedef struct _MMPFNENTRY {
ULONG Modified : 1;
ULONG ReadInProgress : 1;
ULONG WriteInProgress : 1;
ULONG PrototypePte: 1;
ULONG PageColor : 3;
ULONG ParityError : 1;
ULONG PageLocation : 3;
ULONG RemovalRequested : 1;
ULONG CacheAttribute : 2;
ULONG Rom : 1;
ULONG LockCharged : 1; // maintained for DBG only
ULONG DontUse : 16; // overlays USHORT for reference count field.
} MMPFNENTRY;
//
// The cache type definitions are carefully chosen to line up with the
// MEMORY_CACHING_TYPE definitions to ease conversions. Any changes here must
// be reflected throughout the code.
//
typedef enum _MI_PFN_CACHE_ATTRIBUTE {
MiNonCached,
MiCached,
MiWriteCombined,
MiNotMapped
} MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
//
// This conversion array is unfortunately needed because not all
// hardware platforms support all possible cache values. Note that
// the first range is for system RAM, the second range is for I/O space.
//
extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2 * MmMaximumCacheType];
//++
//
// MI_PFN_CACHE_ATTRIBUTE
// MI_TRANSLATE_CACHETYPE (
// IN MEMORY_CACHING_TYPE InputCacheType,
// IN ULONG IoSpace
// )
//
// Routine Description:
//
// Returns the hardware supported cache type for the requested cachetype.
//
// Arguments:
//
// InputCacheType - Supplies the desired cache type.
//
// IoSpace - Supplies nonzero (not necessarily 1 though) if this is
// I/O space, zero if it is main memory.
//
// Return Value:
//
// The actual cache type.
//
//--
FORCEINLINE
MI_PFN_CACHE_ATTRIBUTE
MI_TRANSLATE_CACHETYPE(
IN MEMORY_CACHING_TYPE InputCacheType,
IN ULONG IoSpace
)
{
ASSERT (InputCacheType <= MmWriteCombined);
if (IoSpace != 0) {
IoSpace = MmMaximumCacheType;
}
return MiPlatformCacheAttributes[IoSpace + InputCacheType];
}
//++
//
// VOID
// MI_SET_CACHETYPE_TRANSLATION (
// IN MEMORY_CACHING_TYPE InputCacheType,
// IN ULONG IoSpace,
// IN MI_PFN_CACHE_ATTRIBUTE NewAttribute
// )
//
// Routine Description:
//
// This function sets the hardware supported cachetype for the
// specified cachetype.
//
// Arguments:
//
// InputCacheType - Supplies the desired cache type.
//
// IoSpace - Supplies nonzero (not necessarily 1 though) if this is
// I/O space, zero if it is main memory.
//
// NewAttribute - Supplies the desired attribute.
//
// Return Value:
//
// None.
//
//--
FORCEINLINE
VOID
MI_SET_CACHETYPE_TRANSLATION(
IN MEMORY_CACHING_TYPE InputCacheType,
IN ULONG IoSpace,
IN MI_PFN_CACHE_ATTRIBUTE NewAttribute
)
{
ASSERT (InputCacheType <= MmWriteCombined);
if (IoSpace != 0) {
IoSpace = MmMaximumCacheType;
}
MiPlatformCacheAttributes[IoSpace + InputCacheType] = NewAttribute;
}
#if defined (_X86PAE_)
#pragma pack(1)
#endif
typedef struct _MMPFN {
union {
PFN_NUMBER Flink;
WSLE_NUMBER WsIndex;
PKEVENT Event;
NTSTATUS ReadStatus;
SINGLE_LIST_ENTRY NextStackPfn;
} u1;
PMMPTE PteAddress;
union {
PFN_NUMBER Blink;
ULONG_PTR ShareCount;
} u2;
union {
MMPFNENTRY e1;
struct {
USHORT ShortFlags;
USHORT ReferenceCount;
} e2;
} u3;
#if defined (_WIN64)
ULONG UsedPageTableEntries;
#endif
MMPTE OriginalPte;
union {
ULONG_PTR EntireFrame;
struct {
#if defined (_WIN64)
ULONG_PTR PteFrame: 58;
#else
ULONG_PTR PteFrame: 26;
#endif
ULONG_PTR InPageError : 1;
ULONG_PTR VerifierAllocation : 1;
ULONG_PTR AweAllocation : 1;
ULONG_PTR LockCharged : 1; // maintained for DBG only
ULONG_PTR KernelStack : 1; // only for valid (not trans) pages
ULONG_PTR Reserved : 1;
};
} u4;
} MMPFN, *PMMPFN;
#if defined (_X86PAE_)
#pragma pack()
#endif
// #define _MI_DEBUG_DIRTY 1 // Uncomment this for dirty bit logging
#if defined (_MI_DEBUG_DIRTY)
extern ULONG MiTrackDirtys;
#define MI_DIRTY_BACKTRACE_LENGTH 17
typedef struct _MI_DIRTY_TRACES {
PETHREAD Thread;
PEPROCESS Process;
PMMPFN Pfn;
PMMPTE PointerPte;
ULONG_PTR CallerId;
ULONG_PTR ShareCount;
USHORT ShortFlags;
USHORT ReferenceCount;
PVOID StackTrace [MI_DIRTY_BACKTRACE_LENGTH];
} MI_DIRTY_TRACES, *PMI_DIRTY_TRACES;
extern LONG MiDirtyIndex;
extern PMI_DIRTY_TRACES MiDirtyTraces;
VOID
FORCEINLINE
MiSnapDirty (
IN PMMPFN Pfn,
IN ULONG NewValue,
IN ULONG CallerId
)
{
PEPROCESS Process;
PMI_DIRTY_TRACES Information;
ULONG Index;
ULONG Hash;
if (MiDirtyTraces == NULL) {
return;
}
Index = InterlockedIncrement(&MiDirtyIndex);
Index &= (MiTrackDirtys - 1);
Information = &MiDirtyTraces[Index];
Process = PsGetCurrentProcess ();
Information->Thread = PsGetCurrentThread ();
Information->Process = Process;
Information->Pfn = Pfn;
Information->PointerPte = Pfn->PteAddress;
Information->CallerId = CallerId;
Information->ShareCount = Pfn->u2.ShareCount;
Information->ShortFlags = Pfn->u3.e2.ShortFlags;
Information->ReferenceCount = Pfn->u3.e2.ReferenceCount;
if (NewValue != 0) {
Information->Process = (PEPROCESS) ((ULONG_PTR)Process | 0x1);
}
RtlZeroMemory (&Information->StackTrace[0], MI_DIRTY_BACKTRACE_LENGTH * sizeof(PVOID));
RtlCaptureStackBackTrace (0, MI_DIRTY_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
}
#define MI_SNAP_DIRTY(_Pfn, _NewValue, _Callerid) MiSnapDirty(_Pfn, _NewValue, _Callerid)
#else
#define MI_SNAP_DIRTY(_Pfn, _NewValue, _Callerid)
#endif
#if 0
#define MI_STAMP_MODIFIED(Pfn,id) (Pfn)->u4.Reserved = (id);
#else
#define MI_STAMP_MODIFIED(Pfn,id)
#endif
//++
//
// VOID
// MI_SET_MODIFIED (
// IN PMMPFN Pfn,
// IN ULONG NewValue,
// IN ULONG CallerId
// )
//
// Routine Description:
//
// Set (or clear) the modified bit in the PFN database element.
// The PFN lock must be held.
//
// Arguments:
//
// Pfn - Supplies the PFN to operate on.
//
// NewValue - Supplies 1 to set the modified bit, 0 to clear it.
//
// CallerId - Supplies a caller ID useful for debugging purposes.
//
// Return Value:
//
// None.
//
//--
//
#define MI_SET_MODIFIED(_Pfn, _NewValue, _CallerId) \
MI_SNAP_DIRTY (_Pfn, _NewValue, _CallerId); \
if ((_NewValue) != 0) { \
MI_STAMP_MODIFIED (_Pfn, _CallerId); \
} \
(_Pfn)->u3.e1.Modified = (_NewValue);
//
// ccNUMA is supported in multiprocessor PAE and WIN64 systems only.
//
#if (defined(_WIN64) || defined(_X86PAE_)) && !defined(NT_UP)
#define MI_MULTINODE
VOID
MiDetermineNode (
IN PFN_NUMBER Page,
IN PMMPFN Pfn
);
#else
#define MiDetermineNode(x,y) ((y)->u3.e1.PageColor = 0)
#endif
#if defined (_WIN64)
//
// Note there are some places where these portable macros are not currently
// used because we are not in the correct address space required.
//
#define MI_CAPTURE_USED_PAGETABLE_ENTRIES(PFN) \
ASSERT ((PFN)->UsedPageTableEntries <= PTE_PER_PAGE); \
(PFN)->OriginalPte.u.Soft.UsedPageTableEntries = (PFN)->UsedPageTableEntries;
#define MI_RETRIEVE_USED_PAGETABLE_ENTRIES_FROM_PTE(RBL, PTE) \
ASSERT ((PTE)->u.Soft.UsedPageTableEntries <= PTE_PER_PAGE); \
(RBL)->UsedPageTableEntries = (ULONG)(((PMMPTE)(PTE))->u.Soft.UsedPageTableEntries);
#define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_INPAGE_SUPPORT(INPAGE_SUPPORT) \
(INPAGE_SUPPORT)->UsedPageTableEntries = 0;
#define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_PFN(PFN) (PFN)->UsedPageTableEntries = 0;
#define MI_INSERT_USED_PAGETABLE_ENTRIES_IN_PFN(PFN, INPAGE_SUPPORT) \
ASSERT ((INPAGE_SUPPORT)->UsedPageTableEntries <= PTE_PER_PAGE); \
(PFN)->UsedPageTableEntries = (INPAGE_SUPPORT)->UsedPageTableEntries;
#define MI_ZERO_USED_PAGETABLE_ENTRIES(PFN) \
(PFN)->UsedPageTableEntries = 0;
#define MI_CHECK_USED_PTES_HANDLE(VA) \
ASSERT (MiGetPdeAddress(VA)->u.Hard.Valid == 1);
#define MI_GET_USED_PTES_HANDLE(VA) \
((PVOID)MI_PFN_ELEMENT((PFN_NUMBER)MiGetPdeAddress(VA)->u.Hard.PageFrameNumber))
#define MI_GET_USED_PTES_FROM_HANDLE(PFN) \
((ULONG)(((PMMPFN)(PFN))->UsedPageTableEntries))
#define MI_INCREMENT_USED_PTES_BY_HANDLE(PFN) \
(((PMMPFN)(PFN))->UsedPageTableEntries += 1); \
ASSERT (((PMMPFN)(PFN))->UsedPageTableEntries <= PTE_PER_PAGE)
#define MI_DECREMENT_USED_PTES_BY_HANDLE(PFN) \
(((PMMPFN)(PFN))->UsedPageTableEntries -= 1); \
ASSERT (((PMMPFN)(PFN))->UsedPageTableEntries < PTE_PER_PAGE)
#else
#define MI_CAPTURE_USED_PAGETABLE_ENTRIES(PFN)
#define MI_RETRIEVE_USED_PAGETABLE_ENTRIES_FROM_PTE(RBL, PTE)
#define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_INPAGE_SUPPORT(INPAGE_SUPPORT)
#define MI_ZERO_USED_PAGETABLE_ENTRIES_IN_PFN(PFN)
#define MI_INSERT_USED_PAGETABLE_ENTRIES_IN_PFN(PFN, INPAGE_SUPPORT)
#define MI_CHECK_USED_PTES_HANDLE(VA)
#define MI_GET_USED_PTES_HANDLE(VA) ((PVOID)&MmWorkingSetList->UsedPageTableEntries[MiGetPdeIndex(VA)])
#define MI_GET_USED_PTES_FROM_HANDLE(PDSHORT) ((ULONG)(*(PUSHORT)(PDSHORT)))
#define MI_INCREMENT_USED_PTES_BY_HANDLE(PDSHORT) \
((*(PUSHORT)(PDSHORT)) += 1); \
ASSERT (((*(PUSHORT)(PDSHORT)) <= PTE_PER_PAGE))
#define MI_DECREMENT_USED_PTES_BY_HANDLE(PDSHORT) \
((*(PUSHORT)(PDSHORT)) -= 1); \
ASSERT (((*(PUSHORT)(PDSHORT)) < PTE_PER_PAGE))
#endif
extern PFN_NUMBER MmDynamicPfn;
extern FAST_MUTEX MmDynamicMemoryMutex;
extern PFN_NUMBER MmSystemLockPagesCount;
#if DBG
#define MI_LOCK_ID_COUNTER_MAX 64
ULONG MiLockIds[MI_LOCK_ID_COUNTER_MAX];
#define MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId) \
ASSERT (Pfn->u3.e1.LockCharged == 0); \
ASSERT (CallerId < MI_LOCK_ID_COUNTER_MAX); \
MiLockIds[CallerId] += 1; \
Pfn->u3.e1.LockCharged = 1;
#define MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId) \
ASSERT (Pfn->u3.e1.LockCharged == 1); \
ASSERT (CallerId < MI_LOCK_ID_COUNTER_MAX); \
MiLockIds[CallerId] += 1; \
Pfn->u3.e1.LockCharged = 0;
#else
#define MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId)
#define MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId)
#endif
//++
//
// VOID
// MI_ADD_LOCKED_PAGE_CHARGE (
// IN PMMPFN Pfn
// )
//
// Routine Description:
//
// Charge the systemwide count of locked pages if this is the initial
// lock for this page (multiple concurrent locks are only charged once).
//
// Arguments:
//
// Pfn - the PFN index to operate on.
//
// Return Value:
//
// None.
//
//--
//
#define MI_ADD_LOCKED_PAGE_CHARGE(Pfn, CallerId) \
ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
if (Pfn->u3.e2.ReferenceCount == 1) { \
if (Pfn->u2.ShareCount != 0) { \
ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
MmSystemLockPagesCount += 1; \
} \
else { \
ASSERT (Pfn->u3.e1.LockCharged == 1); \
} \
}
#define MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE(Pfn, CallerId) \
ASSERT (Pfn->u3.e1.PageLocation != ActiveAndValid); \
ASSERT (Pfn->u2.ShareCount == 0); \
if (Pfn->u3.e2.ReferenceCount == 0) { \
MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
MmSystemLockPagesCount += 1; \
}
#define MI_ADD_LOCKED_PAGE_CHARGE_FOR_TRANSITION_PAGE(Pfn, CallerId) \
ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
ASSERT (Pfn->u2.ShareCount == 0); \
ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
if (Pfn->u3.e2.ReferenceCount == 1) { \
MI_MARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
MmSystemLockPagesCount += 1; \
}
//++
//
// VOID
// MI_REMOVE_LOCKED_PAGE_CHARGE (
// IN PMMPFN Pfn
// )
//
// Routine Description:
//
// Remove the charge from the systemwide count of locked pages if this
// is the last lock for this page (multiple concurrent locks are only
// charged once).
//
// The PFN reference checks are carefully ordered so the most common case
// is handled first, the next most common case next, etc. The case of
// a reference count of 2 occurs more than 1000x (yes, 3 orders of
// magnitude) more than a reference count of 1. And reference counts of >2
// occur 3 orders of magnitude more frequently than reference counts of
// exactly 1.
//
// Arguments:
//
// Pfn - the PFN index to operate on.
//
// Return Value:
//
// None.
//
//--
//
#define MI_REMOVE_LOCKED_PAGE_CHARGE(Pfn, CallerId) \
ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
if (Pfn->u3.e2.ReferenceCount == 2) { \
if (Pfn->u2.ShareCount >= 1) { \
ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
MmSystemLockPagesCount -= 1; \
} \
else { \
/* \
* There are multiple referencers to this page and the \
* page is no longer valid in any process address space. \
* The systemwide lock count can only be decremented \
* by the last dereference. \
*/ \
NOTHING; \
} \
} \
else if (Pfn->u3.e2.ReferenceCount != 1) { \
/* \
* There are still multiple referencers to this page (it may \
* or may not be resident in any process address space). \
* Since the systemwide lock count can only be decremented \
* by the last dereference (and this is not it), no action \
* is taken here. \
*/ \
ASSERT (Pfn->u3.e2.ReferenceCount > 2); \
NOTHING; \
} \
else { \
/* \
* This page has already been deleted from all process address \
* spaces. It is sitting in limbo (not on any list) awaiting \
* this last dereference. \
*/ \
ASSERT (Pfn->u3.e2.ReferenceCount == 1); \
ASSERT (Pfn->u3.e1.PageLocation != ActiveAndValid); \
ASSERT (Pfn->u2.ShareCount == 0); \
MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
MmSystemLockPagesCount -= 1; \
}
//++
//
// VOID
// MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (
// IN PMMPFN Pfn
// )
//
// Routine Description:
//
// Remove the charge from the systemwide count of locked pages if this
// is the last lock for this page (multiple concurrent locks are only
// charged once).
//
// The PFN reference checks are carefully ordered so the most common case
// is handled first, the next most common case next, etc. The case of
// a reference count of 2 occurs more than 1000x (yes, 3 orders of
// magnitude) more than a reference count of 1. And reference counts of >2
// occur 3 orders of magnitude more frequently than reference counts of
// exactly 1.
//
// The PFN reference count is then decremented.
//
// Arguments:
//
// Pfn - the PFN index to operate on.
//
// Return Value:
//
// None.
//
//--
//
#define MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF(Pfn, CallerId) \
ASSERT (Pfn->u3.e2.ReferenceCount != 0); \
if (Pfn->u3.e2.ReferenceCount == 2) { \
if (Pfn->u2.ShareCount >= 1) { \
ASSERT (Pfn->u3.e1.PageLocation == ActiveAndValid); \
MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
MmSystemLockPagesCount -= 1; \
} \
else { \
/* \
* There are multiple referencers to this page and the \
* page is no longer valid in any process address space. \
* The systemwide lock count can only be decremented \
* by the last dereference. \
*/ \
NOTHING; \
} \
Pfn->u3.e2.ReferenceCount -= 1; \
} \
else if (Pfn->u3.e2.ReferenceCount != 1) { \
/* \
* There are still multiple referencers to this page (it may \
* or may not be resident in any process address space). \
* Since the systemwide lock count can only be decremented \
* by the last dereference (and this is not it), no action \
* is taken here. \
*/ \
ASSERT (Pfn->u3.e2.ReferenceCount > 2); \
Pfn->u3.e2.ReferenceCount -= 1; \
} \
else { \
/* \
* This page has already been deleted from all process address \
* spaces. It is sitting in limbo (not on any list) awaiting \
* this last dereference. \
*/ \
PFN_NUMBER _PageFrameIndex; \
ASSERT (Pfn->u3.e2.ReferenceCount == 1); \
ASSERT (Pfn->u3.e1.PageLocation != ActiveAndValid); \
ASSERT (Pfn->u2.ShareCount == 0); \
MI_UNMARK_PFN_AS_LOCK_CHARGED(Pfn, CallerId); \
MmSystemLockPagesCount -= 1; \
_PageFrameIndex = Pfn - MmPfnDatabase; \
MiDecrementReferenceCount (_PageFrameIndex); \
}
//++
//
// VOID
// MI_ZERO_WSINDEX (
// IN PMMPFN Pfn
// )
//
// Routine Description:
//
// Zero the Working Set Index field of the argument PFN entry.
// There is a subtlety here on systems where the WsIndex ULONG is
// overlaid with an Event pointer and sizeof(ULONG) != sizeof(PKEVENT).
// Note this will need to be updated if we ever decide to allocate bodies of
// thread objects on 4GB boundaries.
//
// Arguments:
//
// Pfn - the PFN index to operate on.
//
// Return Value:
//
// None.
//
//--
//
#define MI_ZERO_WSINDEX(Pfn) \
Pfn->u1.Event = NULL;
typedef enum _MMSHARE_TYPE {
Normal,
ShareCountOnly,
AndValid
} MMSHARE_TYPE;
typedef struct _MMWSLE_HASH {
PVOID Key;
WSLE_NUMBER Index;
} MMWSLE_HASH, *PMMWSLE_HASH;
//++
//
// WSLE_NUMBER
// MI_WSLE_HASH (
// IN ULONG_PTR VirtualAddress,
// IN PMMWSL WorkingSetList
// )
//
// Routine Description:
//
// Hash the address
//
// Arguments:
//
// VirtualAddress - the address to hash.
//
// WorkingSetList - the working set to hash the address into.
//
// Return Value:
//
// The hash key.
//
//--
//
#define MI_WSLE_HASH(Address, Wsl) \
((WSLE_NUMBER)(((ULONG_PTR)PAGE_ALIGN(Address) >> (PAGE_SHIFT - 2)) % \
((Wsl)->HashTableSize - 1)))
//
// Working Set List Entry.
//
typedef struct _MMWSLENTRY {
ULONG_PTR Valid : 1;
ULONG_PTR LockedInWs : 1;
ULONG_PTR LockedInMemory : 1;
ULONG_PTR Protection : 5;
ULONG_PTR SameProtectAsProto : 1;
ULONG_PTR Direct : 1;
ULONG_PTR Age : 2;
#if MM_VIRTUAL_PAGE_FILLER
ULONG_PTR Filler : MM_VIRTUAL_PAGE_FILLER;
#endif
ULONG_PTR VirtualPageNumber : MM_VIRTUAL_PAGE_SIZE;
} MMWSLENTRY;
typedef struct _MMWSLE {
union {
PVOID VirtualAddress;
ULONG_PTR Long;
MMWSLENTRY e1;
} u1;
} MMWSLE;
#define MI_GET_PROTECTION_FROM_WSLE(Wsl) ((Wsl)->u1.e1.Protection)
typedef MMWSLE *PMMWSLE;
//
// Working Set List. Must be quadword sized.
//
typedef struct _MMWSL {
SIZE_T Quota;
WSLE_NUMBER FirstFree;
WSLE_NUMBER FirstDynamic;
WSLE_NUMBER LastEntry;
WSLE_NUMBER NextSlot; // The next slot to trim
PMMWSLE Wsle;
WSLE_NUMBER LastInitializedWsle;
WSLE_NUMBER NonDirectCount;
PMMWSLE_HASH HashTable;
ULONG HashTableSize;
ULONG NumberOfCommittedPageTables;
PVOID HashTableStart;
PVOID HighestPermittedHashAddress;
ULONG NumberOfImageWaiters;
ULONG VadBitMapHint;
#if (_MI_PAGING_LEVELS >= 4)
PULONG CommittedPageTables;
ULONG NumberOfCommittedPageDirectories;
PULONG CommittedPageDirectories;
ULONG NumberOfCommittedPageDirectoryParents;
ULONG CommittedPageDirectoryParents[(MM_USER_PAGE_DIRECTORY_PARENT_PAGES + sizeof(ULONG)*8-1)/(sizeof(ULONG)*8)];
#elif (_MI_PAGING_LEVELS >= 3)
PULONG CommittedPageTables;
ULONG NumberOfCommittedPageDirectories;
ULONG CommittedPageDirectories[(MM_USER_PAGE_DIRECTORY_PAGES + sizeof(ULONG)*8-1)/(sizeof(ULONG)*8)];
#else
//
// This must be at the end.
// Not used in system cache or session working set lists.
//
USHORT UsedPageTableEntries[MM_USER_PAGE_TABLE_PAGES];
ULONG CommittedPageTables[MM_USER_PAGE_TABLE_PAGES/(sizeof(ULONG)*8)];
#endif
} MMWSL, *PMMWSL;
#if defined(_X86_)
extern PMMWSL MmWorkingSetList;
#endif
extern PKEVENT MiHighMemoryEvent;
extern PKEVENT MiLowMemoryEvent;
//
// The claim estimate of unused pages in a working set is limited
// to grow by this amount per estimation period.
//
#define MI_CLAIM_INCR 30
//
// The maximum number of different ages a page can be.
//
#define MI_USE_AGE_COUNT 4
#define MI_USE_AGE_MAX (MI_USE_AGE_COUNT - 1)
//
// If more than this "percentage" of the working set is estimated to
// be used then allow it to grow freely.
//
#define MI_REPLACEMENT_FREE_GROWTH_SHIFT 5
//
// If more than this "percentage" of the working set has been claimed
// then force replacement in low memory.
//
#define MI_REPLACEMENT_CLAIM_THRESHOLD_SHIFT 3
//
// If more than this "percentage" of the working set is estimated to
// be available then force replacement in low memory.
//
#define MI_REPLACEMENT_EAVAIL_THRESHOLD_SHIFT 3
//
// If while doing replacement a page is found of this age or older then
// replace it. Otherwise the oldest is selected.
//
#define MI_IMMEDIATE_REPLACEMENT_AGE 2
//
// When trimming, use these ages for different passes.
//
#define MI_MAX_TRIM_PASSES 4
#define MI_PASS0_TRIM_AGE 2
#define MI_PASS1_TRIM_AGE 1
#define MI_PASS2_TRIM_AGE 1
#define MI_PASS3_TRIM_AGE 1
#define MI_PASS4_TRIM_AGE 0
//
// If not a forced trim, trim pages older than this age.
//
#define MI_TRIM_AGE_THRESHOLD 2
//
// This "percentage" of a claim is up for grabs in a foreground process.
//
#define MI_FOREGROUND_CLAIM_AVAILABLE_SHIFT 3
//
// This "percentage" of a claim is up for grabs in a background process.
//
#define MI_BACKGROUND_CLAIM_AVAILABLE_SHIFT 1
//++
//
// DWORD
// MI_CALC_NEXT_VALID_ESTIMATION_SLOT (
// DWORD Previous,
// DWORD Minimum,
// DWORD Maximum,
// MI_NEXT_ESTIMATION_SLOT_CONST NextEstimationSlotConst,
// PMMWSLE Wsle
// )
//
// Routine Description:
//
// We iterate through the working set array in a non-sequential
// manner so that the sample is independent of any aging or trimming.
//
// This algorithm walks through the working set with a stride of
// 2^MiEstimationShift elements.
//
// Arguments:
//
// Previous - Last slot used
//
// Minimum - Minimum acceptable slot (ie. the first dynamic one)
//
// Maximum - max slot number + 1
//
// NextEstimationSlotConst - for this algorithm it contains the stride
//
// Wsle - the working set array
//
// Return Value:
//
// Next slot.
//
// Environment:
//
// Kernel mode, APCs disabled, working set lock held and PFN lock held.
//
//--
typedef struct _MI_NEXT_ESTIMATION_SLOT_CONST {
WSLE_NUMBER Stride;
} MI_NEXT_ESTIMATION_SLOT_CONST;
#define MI_CALC_NEXT_ESTIMATION_SLOT_CONST(NextEstimationSlotConst, WorkingSetList) \
(NextEstimationSlotConst).Stride = 1 << MiEstimationShift;
#define MI_NEXT_VALID_ESTIMATION_SLOT(Previous, StartEntry, Minimum, Maximum, NextEstimationSlotConst, Wsle) \
ASSERT(((Previous) >= Minimum) && ((Previous) <= Maximum)); \
ASSERT(((StartEntry) >= Minimum) && ((StartEntry) <= Maximum)); \
do { \
(Previous) += (NextEstimationSlotConst).Stride; \
if ((Previous) > Maximum) { \
(Previous) = Minimum + ((Previous + 1) & (NextEstimationSlotConst.Stride - 1)); \
StartEntry += 1; \
(Previous) = StartEntry; \
} \
if ((Previous) > Maximum || (Previous) < Minimum) { \
StartEntry = Minimum; \
(Previous) = StartEntry; \
} \
} while (Wsle[Previous].u1.e1.Valid == 0);
//++
//
// WSLE_NUMBER
// MI_NEXT_VALID_AGING_SLOT (
// DWORD Previous,
// DWORD Minimum,
// DWORD Maximum,
// PMMWSLE Wsle
// )
//
// Routine Description:
//
// This finds the next slot to valid slot to age. It walks
// through the slots sequentialy.
//
// Arguments:
//
// Previous - Last slot used
//
// Minimum - Minimum acceptable slot (ie. the first dynamic one)
//
// Maximum - Max slot number + 1
//
// Wsle - the working set array
//
// Return Value:
//
// None.
//
// Environment:
//
// Kernel mode, APCs disabled, working set lock held and PFN lock held.
//
//--
#define MI_NEXT_VALID_AGING_SLOT(Previous, Minimum, Maximum, Wsle) \
ASSERT(((Previous) >= Minimum) && ((Previous) <= Maximum)); \
do { \
(Previous) += 1; \
if ((Previous) > Maximum) { \
Previous = Minimum; \
} \
} while ((Wsle[Previous].u1.e1.Valid == 0));
//++
//
// ULONG
// MI_CALCULATE_USAGE_ESTIMATE (
// IN PULONG SampledAgeCounts.
// IN ULONG CounterShift
// )
//
// Routine Description:
//
// In Usage Estimation, we count the number of pages of each age in
// a sample. The function turns the SampledAgeCounts into an
// estimate of the unused pages.
//
// Arguments:
//
// SampledAgeCounts - counts of pages of each different age in the sample
//
// CounterShift - shift necessary to apply sample to entire WS
//
// Return Value:
//
// The number of pages to walk in the working set to get a good
// estimate of the number available.
//
//--
#define MI_CALCULATE_USAGE_ESTIMATE(SampledAgeCounts, CounterShift) \
(((SampledAgeCounts)[1] + \
(SampledAgeCounts)[2] + (SampledAgeCounts)[3]) \
<< (CounterShift))
//++
//
// VOID
// MI_RESET_WSLE_AGE (
// IN PMMPTE PointerPte,
// IN PMMWSLE Wsle
// )
//
// Routine Description:
//
// Clear the age counter for the working set entry.
//
// Arguments:
//
// PointerPte - pointer to the working set list entry's PTE.
//
// Wsle - pointer to the working set list entry.
//
// Return Value:
//
// None.
//
//--
#define MI_RESET_WSLE_AGE(PointerPte, Wsle) \
(Wsle)->u1.e1.Age = 0;
//++
//
// ULONG
// MI_GET_WSLE_AGE (
// IN PMMPTE PointerPte,
// IN PMMWSLE Wsle
// )
//
// Routine Description:
//
// Clear the age counter for the working set entry.
//
// Arguments:
//
// PointerPte - pointer to the working set list entry's PTE
// Wsle - pointer to the working set list entry
//
// Return Value:
//
// Age group of the working set entry
//
//--
#define MI_GET_WSLE_AGE(PointerPte, Wsle) \
((ULONG)((Wsle)->u1.e1.Age))
//++
//
// VOID
// MI_INC_WSLE_AGE (
// IN PMMPTE PointerPte,
// IN PMMWSLE Wsle,
// )
//
// Routine Description:
//
// Increment the age counter for the working set entry.
//
// Arguments:
//
// PointerPte - pointer to the working set list entry's PTE.
//
// Wsle - pointer to the working set list entry.
//
// Return Value:
//
// None
//
//--
#define MI_INC_WSLE_AGE(PointerPte, Wsle) \
if ((Wsle)->u1.e1.Age < 3) { \
(Wsle)->u1.e1.Age += 1; \
}
//++
//
// VOID
// MI_UPDATE_USE_ESTIMATE (
// IN PMMPTE PointerPte,
// IN PMMWSLE Wsle,
// IN ULONG *SampledAgeCounts
// )
//
// Routine Description:
//
// Update the sampled age counts.
//
// Arguments:
//
// PointerPte - pointer to the working set list entry's PTE.
//
// Wsle - pointer to the working set list entry.
//
// SampledAgeCounts - array of age counts to be updated.
//
// Return Value:
//
// None
//
//--
#define MI_UPDATE_USE_ESTIMATE(PointerPte, Wsle, SampledAgeCounts) \
(SampledAgeCounts)[(Wsle)->u1.e1.Age] += 1;
//++
//
// BOOLEAN
// MI_WS_GROWING_TOO_FAST (
// IN PMMSUPPORT VmSupport
// )
//
// Routine Description:
//
// Limit the growth rate of processes as the
// available memory approaches zero. Note the caller must ensure that
// MmAvailablePages is low enough so this calculation does not wrap.
//
// Arguments:
//
// VmSupport - a working set.
//
// Return Value:
//
// TRUE if the growth rate is too fast, FALSE otherwise.
//
//--
#define MI_WS_GROWING_TOO_FAST(VmSupport) \
((VmSupport)->GrowthSinceLastEstimate > \
(((MI_CLAIM_INCR * (MmAvailablePages*MmAvailablePages)) / (64*64)) + 1))
#define SECTION_BASE_ADDRESS(_NtSection) \
(*((PVOID *)&(_NtSection)->PointerToRelocations))
#define SECTION_LOCK_COUNT_POINTER(_NtSection) \
((PLONG)&(_NtSection)->NumberOfRelocations)
//
// Memory Management Object structures.
//
typedef enum _SECTION_CHECK_TYPE {
CheckDataSection,
CheckImageSection,
CheckUserDataSection,
CheckBothSection
} SECTION_CHECK_TYPE;
typedef struct _MMEXTEND_INFO {
UINT64 CommittedSize;
ULONG ReferenceCount;
} MMEXTEND_INFO, *PMMEXTEND_INFO;
typedef struct _SEGMENT {
struct _CONTROL_AREA *ControlArea;
ULONG TotalNumberOfPtes;
ULONG NonExtendedPtes;
ULONG WritableUserReferences;
UINT64 SizeOfSegment;
MMPTE SegmentPteTemplate;
SIZE_T NumberOfCommittedPages;
PMMEXTEND_INFO ExtendInfo;
PVOID SystemImageBase; // could replace with single bit in ldrmodlist
PVOID BasedAddress;
//
// The fields below are for image & pagefile-backed sections only.
// Common fields are above and new common entries must be added to
// both the SEGMENT and MAPPED_FILE_SEGMENT declarations.
//
union {
SIZE_T ImageCommitment; // for image-backed sections only
PEPROCESS CreatingProcess; // for pagefile-backed sections only
} u1;
union {
PSECTION_IMAGE_INFORMATION ImageInformation; // for images only
PVOID FirstMappedVa; // for pagefile-backed sections only
} u2;
PMMPTE PrototypePte;
MMPTE ThePtes[MM_PROTO_PTE_ALIGNMENT / PAGE_SIZE];
} SEGMENT, *PSEGMENT;
typedef struct _MAPPED_FILE_SEGMENT {
struct _CONTROL_AREA *ControlArea;
ULONG TotalNumberOfPtes;
ULONG NonExtendedPtes;
ULONG WritableUserReferences;
UINT64 SizeOfSegment;
MMPTE SegmentPteTemplate;
SIZE_T NumberOfCommittedPages;
PMMEXTEND_INFO ExtendInfo;
PVOID SystemImageBase; // could replace with single bit in ldrmodlist
PVOID BasedAddress;
struct _MSUBSECTION *LastSubsectionHint;
} MAPPED_FILE_SEGMENT, *PMAPPED_FILE_SEGMENT;
typedef struct _EVENT_COUNTER {
SINGLE_LIST_ENTRY ListEntry;
ULONG RefCount;
KEVENT Event;
} EVENT_COUNTER, *PEVENT_COUNTER;
typedef struct _MMSECTION_FLAGS {
unsigned BeingDeleted : 1;
unsigned BeingCreated : 1;
unsigned BeingPurged : 1;
unsigned NoModifiedWriting : 1;
unsigned FailAllIo : 1;
unsigned Image : 1;
unsigned Based : 1;
unsigned File : 1;
unsigned Networked : 1;
unsigned NoCache : 1;
unsigned PhysicalMemory : 1;
unsigned CopyOnWrite : 1;
unsigned Reserve : 1; // not a spare bit!
unsigned Commit : 1;
unsigned FloppyMedia : 1;
unsigned WasPurged : 1;
unsigned UserReference : 1;
unsigned GlobalMemory : 1;
unsigned DeleteOnClose : 1;
unsigned FilePointerNull : 1;
unsigned DebugSymbolsLoaded : 1;
unsigned SetMappedFileIoComplete : 1;
unsigned CollidedFlush : 1;
unsigned NoChange : 1;
unsigned HadUserReference : 1;
unsigned ImageMappedInSystemSpace : 1;
unsigned UserWritable : 1;
unsigned Accessed : 1;
unsigned GlobalOnlyPerSession : 1;
unsigned Rom : 1;
unsigned filler : 2;
} MMSECTION_FLAGS;
typedef struct _CONTROL_AREA { // must be quadword sized.
PSEGMENT Segment;
LIST_ENTRY DereferenceList;
ULONG NumberOfSectionReferences;
ULONG NumberOfPfnReferences;
ULONG NumberOfMappedViews;
USHORT NumberOfSubsections;
USHORT FlushInProgressCount;
ULONG NumberOfUserReferences;
union {
ULONG LongFlags;
MMSECTION_FLAGS Flags;
} u;
PFILE_OBJECT FilePointer;
PEVENT_COUNTER WaitingForDeletion;
USHORT ModifiedWriteCount;
USHORT NumberOfSystemCacheViews;
} CONTROL_AREA, *PCONTROL_AREA;
typedef struct _LARGE_CONTROL_AREA { // must be quadword sized.
PSEGMENT Segment;
LIST_ENTRY DereferenceList;
ULONG NumberOfSectionReferences;
ULONG NumberOfPfnReferences;
ULONG NumberOfMappedViews;
USHORT NumberOfSubsections;
USHORT FlushInProgressCount;
ULONG NumberOfUserReferences;
union {
ULONG LongFlags;
MMSECTION_FLAGS Flags;
} u;
PFILE_OBJECT FilePointer;
PEVENT_COUNTER WaitingForDeletion;
USHORT ModifiedWriteCount;
USHORT NumberOfSystemCacheViews;
PFN_NUMBER StartingFrame; // only used if Flags.Rom == 1.
LIST_ENTRY UserGlobalList;
ULONG SessionId;
} LARGE_CONTROL_AREA, *PLARGE_CONTROL_AREA;
typedef struct _MMSUBSECTION_FLAGS {
unsigned ReadOnly : 1;
unsigned ReadWrite : 1;
unsigned SubsectionStatic : 1;
unsigned GlobalMemory: 1;
unsigned Protection : 5;
unsigned LargePages : 1;
unsigned StartingSector4132 : 10; // 2 ** (42+12) == 4MB*4GB == 16K TB
unsigned SectorEndOffset : 12;
} MMSUBSECTION_FLAGS;
typedef struct _SUBSECTION { // Must start on quadword boundary and be quad sized
PCONTROL_AREA ControlArea;
union {
ULONG LongFlags;
MMSUBSECTION_FLAGS SubsectionFlags;
} u;
ULONG StartingSector;
ULONG NumberOfFullSectors; // (4GB-1) * 4K == 16TB-4K limit per subsection
PMMPTE SubsectionBase;
ULONG UnusedPtes;
ULONG PtesInSubsection;
struct _SUBSECTION *NextSubsection;
} SUBSECTION, *PSUBSECTION;
extern const ULONG MMSECT;
//
// Accesses to MMSUBSECTION_FLAGS2 are synchronized via the PFN lock
// (unlike MMSUBSECTION_FLAGS access which is not lock protected at all).
//
typedef struct _MMSUBSECTION_FLAGS2 {
unsigned SubsectionAccessed : 1;
unsigned SubsectionConverted : 1; // only needed for debug
unsigned Reserved : 30;
} MMSUBSECTION_FLAGS2;
//
// Mapped data file subsection structure. Not used for images
// or pagefile-backed shared memory.
//
typedef struct _MSUBSECTION { // Must start on quadword boundary and be quad sized
PCONTROL_AREA ControlArea;
union {
ULONG LongFlags;
MMSUBSECTION_FLAGS SubsectionFlags;
} u;
ULONG StartingSector;
ULONG NumberOfFullSectors; // (4GB-1) * 4K == 16TB-4K limit per subsection
PMMPTE SubsectionBase;
ULONG UnusedPtes;
ULONG PtesInSubsection;
struct _SUBSECTION *NextSubsection;
LIST_ENTRY DereferenceList;
ULONG_PTR NumberOfMappedViews;
union {
ULONG LongFlags2;
MMSUBSECTION_FLAGS2 SubsectionFlags2;
} u2;
} MSUBSECTION, *PMSUBSECTION;
#define MI_MAXIMUM_SECTION_SIZE ((UINT64)16*1024*1024*1024*1024*1024 - (1<<MM4K_SHIFT))
VOID
MiDecrementSubsections (
IN PSUBSECTION FirstSubsection,
IN PSUBSECTION LastSubsection OPTIONAL
);
NTSTATUS
MiAddViewsForSectionWithPfn (
IN PMSUBSECTION StartMappedSubsection,
IN ULONG LastPteOffset OPTIONAL
);
NTSTATUS
MiAddViewsForSection (
IN PMSUBSECTION MappedSubsection,
IN ULONG LastPteOffset OPTIONAL,
IN KIRQL OldIrql,
OUT PULONG Waited
);
LOGICAL
MiReferenceSubsection (
IN PMSUBSECTION MappedSubsection
);
VOID
MiRemoveViewsFromSection (
IN PMSUBSECTION StartMappedSubsection,
IN ULONG LastPteOffset OPTIONAL
);
VOID
MiRemoveViewsFromSectionWithPfn (
IN PMSUBSECTION StartMappedSubsection,
IN ULONG LastPteOffset OPTIONAL
);
VOID
MiSubsectionConsistent(
IN PSUBSECTION Subsection
);
#if DBG
#define MI_CHECK_SUBSECTION(_subsection) MiSubsectionConsistent((PSUBSECTION)(_subsection))
#else
#define MI_CHECK_SUBSECTION(_subsection)
#endif
//++
//ULONG
//Mi4KStartForSubsection (
// IN PLARGE_INTEGER address,
// IN OUT PSUBSECTION subsection
// );
//
// Routine Description:
//
// This macro sets into the specified subsection the supplied information
// indicating the start address (in 4K units) of this portion of the file.
//
// Arguments
//
// address - Supplies the 64-bit address (in 4K units) of the start of this
// portion of the file.
//
// subsection - Supplies the subsection address to store the address in.
//
// Return Value:
//
// None.
//
//--
#define Mi4KStartForSubsection(address, subsection) \
subsection->StartingSector = ((PLARGE_INTEGER)address)->LowPart; \
subsection->u.SubsectionFlags.StartingSector4132 = \
(((PLARGE_INTEGER)(address))->HighPart & 0x3ff);
//++
//ULONG
//Mi4KStartFromSubsection (
// IN OUT PLARGE_INTEGER address,
// IN PSUBSECTION subsection
// );
//
// Routine Description:
//
// This macro gets the start 4K offset from the specified subsection.
//
// Arguments
//
// address - Supplies the 64-bit address (in 4K units) to place the
// start of this subsection into.
//
// subsection - Supplies the subsection address to get the address from.
//
// Return Value:
//
// None.
//
//--
#define Mi4KStartFromSubsection(address, subsection) \
((PLARGE_INTEGER)address)->LowPart = subsection->StartingSector; \
((PLARGE_INTEGER)address)->HighPart = subsection->u.SubsectionFlags.StartingSector4132;
typedef struct _MMDEREFERENCE_SEGMENT_HEADER {
KSPIN_LOCK Lock;
KSEMAPHORE Semaphore;
LIST_ENTRY ListHead;
} MMDEREFERENCE_SEGMENT_HEADER;
//
// This entry is used for calling the segment dereference thread
// to perform page file expansion. It has a similar structure
// to a control area to allow either a control area or a page file
// expansion entry to be placed on the list. Note that for a control
// area the segment pointer is valid whereas for page file expansion
// it is null.
//
typedef struct _MMPAGE_FILE_EXPANSION {
PSEGMENT Segment;
LIST_ENTRY DereferenceList;
SIZE_T RequestedExpansionSize;
SIZE_T ActualExpansion;
KEVENT Event;
LONG InProgress;
ULONG PageFileNumber;
} MMPAGE_FILE_EXPANSION, *PMMPAGE_FILE_EXPANSION;
#define MI_EXTEND_ANY_PAGEFILE ((ULONG)-1)
#define MI_CONTRACT_PAGEFILES ((SIZE_T)-1)
typedef struct _MMWORKING_SET_EXPANSION_HEAD {
LIST_ENTRY ListHead;
} MMWORKING_SET_EXPANSION_HEAD;
#define SUBSECTION_READ_ONLY 1L
#define SUBSECTION_READ_WRITE 2L
#define SUBSECTION_COPY_ON_WRITE 4L
#define SUBSECTION_SHARE_ALLOW 8L
//
// The MMINPAGE_FLAGS relies on the fact that a pool allocation is always
// QUADWORD aligned so the low 3 bits are always available.
//
typedef struct _MMINPAGE_FLAGS {
ULONG_PTR Completed : 1;
ULONG_PTR Available1 : 1;
ULONG_PTR Available2 : 1;
#if defined (_WIN64)
ULONG_PTR PrefetchMdlHighBits : 61;
#else
ULONG_PTR PrefetchMdlHighBits : 29;
#endif
} MMINPAGE_FLAGS, *PMMINPAGE_FLAGS;
#define MI_EXTRACT_PREFETCH_MDL(_Support) ((PMDL)((ULONG_PTR)(_Support->u1.PrefetchMdl) & ~(sizeof(QUAD) - 1)))
typedef struct _MMINPAGE_SUPPORT {
KEVENT Event;
IO_STATUS_BLOCK IoStatus;
LARGE_INTEGER ReadOffset;
LONG WaitCount;
#if defined (_WIN64)
ULONG UsedPageTableEntries;
#endif
PETHREAD Thread;
PFILE_OBJECT FilePointer;
PMMPTE BasePte;
PMMPFN Pfn;
union {
MMINPAGE_FLAGS e1;
ULONG_PTR LongFlags;
PMDL PrefetchMdl; // Only used under _PREFETCH_
} u1;
MDL Mdl;
PFN_NUMBER Page[MM_MAXIMUM_READ_CLUSTER_SIZE + 1];
SINGLE_LIST_ENTRY ListEntry;
} MMINPAGE_SUPPORT, *PMMINPAGE_SUPPORT;
#define MI_PF_DUMMY_PAGE_PTE ((PMMPTE)0x23452345) // Only used by _PREFETCH_
//
// Address Node.
//
typedef struct _MMADDRESS_NODE {
ULONG_PTR StartingVpn;
ULONG_PTR EndingVpn;
struct _MMADDRESS_NODE *Parent;
struct _MMADDRESS_NODE *LeftChild;
struct _MMADDRESS_NODE *RightChild;
} MMADDRESS_NODE, *PMMADDRESS_NODE;
typedef struct _SECTION {
MMADDRESS_NODE Address;
PSEGMENT Segment;
LARGE_INTEGER SizeOfSection;
union {
ULONG LongFlags;
MMSECTION_FLAGS Flags;
} u;
ULONG InitialPageProtection;
} SECTION, *PSECTION;
//
// Banked memory descriptor. Pointed to by VAD which has
// the PhysicalMemory flags set and the Banked pointer field as
// non-NULL.
//
typedef struct _MMBANKED_SECTION {
PFN_NUMBER BasePhysicalPage;
PMMPTE BasedPte;
ULONG BankSize;
ULONG BankShift; //shift for PTEs to calculate bank number
PBANKED_SECTION_ROUTINE BankedRoutine;
PVOID Context;
PMMPTE CurrentMappedPte;
MMPTE BankTemplate[1];
} MMBANKED_SECTION, *PMMBANKED_SECTION;
//
// Virtual address descriptor
//
// ***** NOTE **********
// The first part of a virtual address descriptor is a MMADDRESS_NODE!!!
//
#if defined (_WIN64)
#define COMMIT_SIZE 51
#if ((COMMIT_SIZE + PAGE_SHIFT) < 63)
#error COMMIT_SIZE too small
#endif
#else
#define COMMIT_SIZE 19
#if ((COMMIT_SIZE + PAGE_SHIFT) < 31)
#error COMMIT_SIZE too small
#endif
#endif
#define MM_MAX_COMMIT (((ULONG_PTR) 1 << COMMIT_SIZE) - 1)
#define MM_VIEW_UNMAP 0
#define MM_VIEW_SHARE 1
typedef struct _MMVAD_FLAGS {
ULONG_PTR CommitCharge : COMMIT_SIZE; //limits system to 4k pages or bigger!
ULONG_PTR PhysicalMapping : 1;
ULONG_PTR ImageMap : 1;
ULONG_PTR UserPhysicalPages : 1;
ULONG_PTR NoChange : 1;
ULONG_PTR WriteWatch : 1;
ULONG_PTR Protection : 5;
ULONG_PTR LargePages : 1;
ULONG_PTR MemCommit: 1;
ULONG_PTR PrivateMemory : 1; //used to tell VAD from VAD_SHORT
} MMVAD_FLAGS;
typedef struct _MMVAD_FLAGS2 {
unsigned FileOffset : 24; // number of 64k units into file
unsigned SecNoChange : 1; // set if SEC_NOCHANGE specified
unsigned OneSecured : 1; // set if u3 field is a range
unsigned MultipleSecured : 1; // set if u3 field is a list head
unsigned ReadOnly : 1; // protected as ReadOnly
unsigned LongVad : 1; // set if VAD is a long VAD
unsigned ExtendableFile : 1;
unsigned Inherit : 1; //1 = ViewShare, 0 = ViewUnmap
unsigned CopyOnWrite : 1;
} MMVAD_FLAGS2;
typedef struct _MMADDRESS_LIST {
ULONG_PTR StartVpn;
ULONG_PTR EndVpn;
} MMADDRESS_LIST, *PMMADDRESS_LIST;
typedef struct _MMSECURE_ENTRY {
union {
ULONG_PTR LongFlags2;
MMVAD_FLAGS2 VadFlags2;
} u2;
ULONG_PTR StartVpn;
ULONG_PTR EndVpn;
LIST_ENTRY List;
} MMSECURE_ENTRY, *PMMSECURE_ENTRY;
typedef struct _ALIAS_VAD_INFO {
KAPC Apc;
ULONG NumberOfEntries;
ULONG MaximumEntries;
} ALIAS_VAD_INFO, *PALIAS_VAD_INFO;
typedef struct _ALIAS_VAD_INFO2 {
ULONG BaseAddress;
HANDLE SecureHandle;
} ALIAS_VAD_INFO2, *PALIAS_VAD_INFO2;
typedef struct _MMVAD {
ULONG_PTR StartingVpn;
ULONG_PTR EndingVpn;
struct _MMVAD *Parent;
struct _MMVAD *LeftChild;
struct _MMVAD *RightChild;
union {
ULONG_PTR LongFlags;
MMVAD_FLAGS VadFlags;
} u;
PCONTROL_AREA ControlArea;
PMMPTE FirstPrototypePte;
PMMPTE LastContiguousPte;
union {
ULONG LongFlags2;
MMVAD_FLAGS2 VadFlags2;
} u2;
} MMVAD, *PMMVAD;
typedef struct _MMVAD_LONG {
ULONG_PTR StartingVpn;
ULONG_PTR EndingVpn;
struct _MMVAD *Parent;
struct _MMVAD *LeftChild;
struct _MMVAD *RightChild;
union {
ULONG_PTR LongFlags;
MMVAD_FLAGS VadFlags;
} u;
PCONTROL_AREA ControlArea;
PMMPTE FirstPrototypePte;
PMMPTE LastContiguousPte;
union {
ULONG LongFlags2;
MMVAD_FLAGS2 VadFlags2;
} u2;
union {
LIST_ENTRY List;
MMADDRESS_LIST Secured;
} u3;
union {
PMMBANKED_SECTION Banked;
PMMEXTEND_INFO ExtendedInfo;
} u4;
#if defined(_MIALT4K_)
PALIAS_VAD_INFO AliasInformation;
#endif
} MMVAD_LONG, *PMMVAD_LONG;
typedef struct _MMVAD_SHORT {
ULONG_PTR StartingVpn;
ULONG_PTR EndingVpn;
struct _MMVAD *Parent;
struct _MMVAD *LeftChild;
struct _MMVAD *RightChild;
union {
ULONG_PTR LongFlags;
MMVAD_FLAGS VadFlags;
} u;
} MMVAD_SHORT, *PMMVAD_SHORT;
#define MI_GET_PROTECTION_FROM_VAD(_Vad) ((ULONG)(_Vad)->u.VadFlags.Protection)
#define MI_PHYSICAL_VIEW_AWE 0x1 // AWE region
#define MI_PHYSICAL_VIEW_PHYS 0x2 // Device\PhysicalMemory region
typedef struct _MI_PHYSICAL_VIEW {
LIST_ENTRY ListEntry;
PMMVAD Vad;
PCHAR StartVa;
PCHAR EndVa;
union {
ULONG_PTR LongFlags; // physical or AWE Vad identification
PRTL_BITMAP BitMap; // only if Vad->u.VadFlags.WriteWatch == 1
} u;
} MI_PHYSICAL_VIEW, *PMI_PHYSICAL_VIEW;
#define MI_PHYSICAL_VIEW_KEY 'vpmM'
#define MI_WRITEWATCH_VIEW_KEY 'wWmM'
//
// Stuff for support of Write Watch.
//
extern ULONG_PTR MiActiveWriteWatch;
VOID
MiCaptureWriteWatchDirtyBit (
IN PEPROCESS Process,
IN PVOID VirtualAddress
);
//
// Stuff for support of AWE (Address Windowing Extensions).
//
typedef struct _AWEINFO {
PRTL_BITMAP VadPhysicalPagesBitMap;
ULONG_PTR VadPhysicalPages;
//
// The PushLock is used to allow most of the NtMapUserPhysicalPages{Scatter}
// to execute in parallel as this is acquired shared for these calls.
// Exclusive acquisitions are used to protect maps against frees of the
// pages as well as to protect updates to the AweVadList. Collisions
// should be rare because the exclusive acquisitions should be rare.
//
PEX_PUSH_LOCK_CACHE_AWARE PushLock;
LIST_ENTRY AweVadList;
} AWEINFO, *PAWEINFO;
VOID
MiAweViewInserter (
IN PEPROCESS Process,
IN PMI_PHYSICAL_VIEW PhysicalView
);
VOID
MiAweViewRemover (
IN PEPROCESS Process,
IN PMMVAD Vad
);
//
// Stuff for support of POSIX Fork.
//
typedef struct _MMCLONE_BLOCK {
MMPTE ProtoPte;
LONG CloneRefCount;
} MMCLONE_BLOCK, *PMMCLONE_BLOCK;
typedef struct _MMCLONE_HEADER {
ULONG NumberOfPtes;
LONG NumberOfProcessReferences;
PMMCLONE_BLOCK ClonePtes;
} MMCLONE_HEADER, *PMMCLONE_HEADER;
typedef struct _MMCLONE_DESCRIPTOR {
ULONG_PTR StartingVpn;
ULONG_PTR EndingVpn;
struct _MMCLONE_DESCRIPTOR *Parent;
struct _MMCLONE_DESCRIPTOR *LeftChild;
struct _MMCLONE_DESCRIPTOR *RightChild;
ULONG NumberOfPtes;
PMMCLONE_HEADER CloneHeader;
LONG NumberOfReferences;
LONG FinalNumberOfReferences;
SIZE_T PagedPoolQuotaCharge;
} MMCLONE_DESCRIPTOR, *PMMCLONE_DESCRIPTOR;
//
// The following macro allocates and initializes a bitmap from the
// specified pool of the specified size.
//
// VOID
// MiCreateBitMap (
// OUT PRTL_BITMAP *BitMapHeader,
// IN SIZE_T SizeOfBitMap,
// IN POOL_TYPE PoolType
// );
//
#define MiCreateBitMap(BMH,S,P) { \
ULONG _S; \
ASSERT ((ULONG64)(S) < _4gb); \
_S = sizeof(RTL_BITMAP) + (ULONG)((((S) + 31) / 32) * 4); \
*(BMH) = (PRTL_BITMAP)ExAllocatePoolWithTag( (P), _S, ' mM'); \
if (*(BMH)) { \
RtlInitializeBitMap( *(BMH), (PULONG)((*(BMH))+1), (ULONG)(S)); \
} \
}
#define MiRemoveBitMap(BMH) { \
ExFreePool(*(BMH)); \
*(BMH) = NULL; \
}
#define MI_INITIALIZE_ZERO_MDL(MDL) { \
MDL->Next = (PMDL) NULL; \
MDL->MdlFlags = 0; \
MDL->StartVa = NULL; \
MDL->ByteOffset = 0; \
MDL->ByteCount = 0; \
}
//
// Page File structures.
//
typedef struct _MMMOD_WRITER_LISTHEAD {
LIST_ENTRY ListHead;
KEVENT Event;
} MMMOD_WRITER_LISTHEAD, *PMMMOD_WRITER_LISTHEAD;
typedef struct _MMMOD_WRITER_MDL_ENTRY {
LIST_ENTRY Links;
LARGE_INTEGER WriteOffset;
union {
IO_STATUS_BLOCK IoStatus;
LARGE_INTEGER LastByte;
} u;
PIRP Irp;
ULONG_PTR LastPageToWrite;
PMMMOD_WRITER_LISTHEAD PagingListHead;
PLIST_ENTRY CurrentList;
struct _MMPAGING_FILE *PagingFile;
PFILE_OBJECT File;
PCONTROL_AREA ControlArea;
PERESOURCE FileResource;
MDL Mdl;
PFN_NUMBER Page[1];
} MMMOD_WRITER_MDL_ENTRY, *PMMMOD_WRITER_MDL_ENTRY;
#define MM_PAGING_FILE_MDLS 2
typedef struct _MMPAGING_FILE {
PFN_NUMBER Size;
PFN_NUMBER MaximumSize;
PFN_NUMBER MinimumSize;
PFN_NUMBER FreeSpace;
PFN_NUMBER CurrentUsage;
PFN_NUMBER PeakUsage;
PFN_NUMBER Hint;
PFN_NUMBER HighestPage;
PMMMOD_WRITER_MDL_ENTRY Entry[MM_PAGING_FILE_MDLS];
PRTL_BITMAP Bitmap;
PFILE_OBJECT File;
UNICODE_STRING PageFileName;
ULONG PageFileNumber;
BOOLEAN Extended;
BOOLEAN HintSetToZero;
BOOLEAN BootPartition;
HANDLE FileHandle;
} MMPAGING_FILE, *PMMPAGING_FILE;
//
// System PTE structures.
//
typedef enum _MMSYSTEM_PTE_POOL_TYPE {
SystemPteSpace,
NonPagedPoolExpansion,
MaximumPtePoolTypes
} MMSYSTEM_PTE_POOL_TYPE;
typedef struct _MMFREE_POOL_ENTRY {
LIST_ENTRY List; // maintained free&chk, 1st entry only
PFN_NUMBER Size; // maintained free&chk, 1st entry only
ULONG Signature; // maintained chk only, all entries
struct _MMFREE_POOL_ENTRY *Owner; // maintained free&chk, all entries
} MMFREE_POOL_ENTRY, *PMMFREE_POOL_ENTRY;
typedef struct _MMLOCK_CONFLICT {
LIST_ENTRY List;
PETHREAD Thread;
} MMLOCK_CONFLICT, *PMMLOCK_CONFLICT;
//
// System view structures
//
typedef struct _MMVIEW {
ULONG_PTR Entry;
PCONTROL_AREA ControlArea;
} MMVIEW, *PMMVIEW;
//
// The MMSESSION structure represents kernel memory that is only valid on a
// per-session basis, thus the calling thread must be in the proper session
// to access this structure.
//
typedef struct _MMSESSION {
//
// Never refer to the SystemSpaceViewLock directly - always use the pointer
// following it or you will break support for multiple concurrent sessions.
//
FAST_MUTEX SystemSpaceViewLock;
//
// This points to the mutex above and is needed because the MMSESSION
// is mapped in session space on Hydra and the mutex needs to be globally
// visible for proper KeWaitForSingleObject & KeSetEvent operation.
//
PFAST_MUTEX SystemSpaceViewLockPointer;
PCHAR SystemSpaceViewStart;
PMMVIEW SystemSpaceViewTable;
ULONG SystemSpaceHashSize;
ULONG SystemSpaceHashEntries;
ULONG SystemSpaceHashKey;
PRTL_BITMAP SystemSpaceBitMap;
} MMSESSION, *PMMSESSION;
extern MMSESSION MmSession;
#define LOCK_SYSTEM_VIEW_SPACE(_Session) \
ExAcquireFastMutex (_Session->SystemSpaceViewLockPointer)
#define UNLOCK_SYSTEM_VIEW_SPACE(_Session) \
ExReleaseFastMutex (_Session->SystemSpaceViewLockPointer)
//
// List for flushing TBs singularly.
//
typedef struct _MMPTE_FLUSH_LIST {
ULONG Count;
PMMPTE FlushPte[MM_MAXIMUM_FLUSH_COUNT];
PVOID FlushVa[MM_MAXIMUM_FLUSH_COUNT];
} MMPTE_FLUSH_LIST, *PMMPTE_FLUSH_LIST;
typedef struct _LOCK_TRACKER {
LIST_ENTRY ListEntry;
PMDL Mdl;
PVOID StartVa;
PFN_NUMBER Count;
ULONG Offset;
ULONG Length;
PFN_NUMBER Page;
PVOID CallingAddress;
PVOID CallersCaller;
LIST_ENTRY GlobalListEntry;
ULONG Who;
PEPROCESS Process;
} LOCK_TRACKER, *PLOCK_TRACKER;
extern LOGICAL MmTrackLockedPages;
extern BOOLEAN MiTrackingAborted;
extern KSPIN_LOCK MiTrackLockedPagesLock;
typedef struct _LOCK_HEADER {
LIST_ENTRY ListHead;
PFN_NUMBER Count;
} LOCK_HEADER, *PLOCK_HEADER;
extern LOGICAL MmSnapUnloads;
#define MI_UNLOADED_DRIVERS 50
extern ULONG MmLastUnloadedDriver;
extern PUNLOADED_DRIVERS MmUnloadedDrivers;
VOID
MiInitMachineDependent (
IN PLOADER_PARAMETER_BLOCK LoaderBlock
);
VOID
MiReportPhysicalMemory (
VOID
);
extern PFN_NUMBER MiNumberOfCompressionPages;
NTSTATUS
MiArmCompressionInterrupt (
VOID
);
VOID
MiBuildPagedPool (
VOID
);
VOID
MiInitializeNonPagedPool (
VOID
);
LOGICAL
MiInitializeSystemSpaceMap (
PVOID Session OPTIONAL
);
VOID
MiFindInitializationCode (
OUT PVOID *StartVa,
OUT PVOID *EndVa
);
VOID
MiFreeInitializationCode (
IN PVOID StartVa,
IN PVOID EndVa
);
extern ULONG MiNonCachedCollisions;
//
// If /NOLOWMEM is used, this is set to the boundary PFN (pages below this
// value are not used whenever possible).
//
extern PFN_NUMBER MiNoLowMemory;
PVOID
MiAllocateLowMemory (
IN SIZE_T NumberOfBytes,
IN PFN_NUMBER LowestAcceptablePfn,
IN PFN_NUMBER HighestAcceptablePfn,
IN PFN_NUMBER BoundaryPfn,
IN PVOID CallingAddress,
IN MEMORY_CACHING_TYPE CacheType,
IN ULONG Tag
);
LOGICAL
MiFreeLowMemory (
IN PVOID BaseAddress,
IN ULONG Tag
);
//
// Move drivers out of the low 16mb that ntldr placed them at - this makes more
// memory below 16mb available for ISA-type drivers that cannot run without it.
//
extern LOGICAL MmMakeLowMemory;
VOID
MiRemoveLowPages (
IN ULONG RemovePhase
);
ULONG
MiSectionInitialization (
VOID
);
#define MI_MAX_DEREFERENCE_CHUNK (64 * 1024 / PAGE_SIZE)
typedef struct _MI_PFN_DEREFERENCE_CHUNK {
SINGLE_LIST_ENTRY ListEntry;
CSHORT Flags;
USHORT NumberOfPages;
PFN_NUMBER Pfns[MI_MAX_DEREFERENCE_CHUNK];
} MI_PFN_DEREFERENCE_CHUNK, *PMI_PFN_DEREFERENCE_CHUNK;
extern SLIST_HEADER MmPfnDereferenceSListHead;
extern PSINGLE_LIST_ENTRY MmPfnDeferredList;
#define MI_DEFER_PFN_HELD 0x1
#define MI_DEFER_DRAIN_LOCAL_ONLY 0x2
VOID
MiDeferredUnlockPages (
ULONG Flags
);
LOGICAL
MiFreeAllExpansionNonPagedPool (
IN LOGICAL PoolLockHeld
);
VOID
FASTCALL
MiDecrementReferenceCount (
IN PFN_NUMBER PageFrameIndex
);
//++
//VOID
//MiDecrementReferenceCountInline (
// IN PMMPFN PFN
// IN PFN_NUMBER FRAME
// );
//
// Routine Description:
//
// MiDecrementReferenceCountInline decrements the reference count inline,
// only calling MiDecrementReferenceCount if the count would go to zero
// which would cause the page to be released.
//
// Arguments:
//
// PFN - Supplies the PFN to decrement.
//
// FRAME - Supplies the frame matching the above PFN.
//
// Return Value:
//
// None.
//
// Environment:
//
// PFN lock held.
//
//--
#define MiDecrementReferenceCountInline(PFN, FRAME) \
MM_PFN_LOCK_ASSERT(); \
ASSERT (MI_PFN_ELEMENT(FRAME) == (PFN)); \
ASSERT ((FRAME) <= MmHighestPhysicalPage); \
ASSERT ((PFN)->u3.e2.ReferenceCount != 0); \
if ((PFN)->u3.e2.ReferenceCount != 1) { \
(PFN)->u3.e2.ReferenceCount -= 1; \
} \
else { \
MiDecrementReferenceCount (FRAME); \
}
VOID
FASTCALL
MiDecrementShareCount (
IN PFN_NUMBER PageFrameIndex
);
#define MiDecrementShareCountOnly(P) MiDecrementShareCount(P)
#define MiDecrementShareAndValidCount(P) MiDecrementShareCount(P)
//++
//VOID
//MiDecrementShareCountInline (
// IN PMMPFN PFN,
// IN PFN_NUMBER FRAME
// );
//
// Routine Description:
//
// MiDecrementShareCountInline decrements the share count inline,
// only calling MiDecrementShareCount if the count would go to zero
// which would cause the page to be released.
//
// Arguments:
//
// PFN - Supplies the PFN to decrement.
//
// FRAME - Supplies the frame matching the above PFN.
//
// Return Value:
//
// None.
//
// Environment:
//
// PFN lock held.
//
//--
#define MiDecrementShareCountInline(PFN, FRAME) \
MM_PFN_LOCK_ASSERT(); \
ASSERT (((FRAME) <= MmHighestPhysicalPage) && ((FRAME) > 0)); \
ASSERT (MI_PFN_ELEMENT(FRAME) == (PFN)); \
ASSERT ((PFN)->u2.ShareCount != 0); \
if ((PFN)->u3.e1.PageLocation != ActiveAndValid && (PFN)->u3.e1.PageLocation != StandbyPageList) { \
KeBugCheckEx (PFN_LIST_CORRUPT, 0x99, FRAME, (PFN)->u3.e1.PageLocation, 0); \
} \
if ((PFN)->u2.ShareCount != 1) { \
(PFN)->u2.ShareCount -= 1; \
PERFINFO_DECREFCNT((PFN), PERF_SOFT_TRIM, PERFINFO_LOG_TYPE_DECSHARCNT); \
ASSERT ((PFN)->u2.ShareCount < 0xF000000); \
} \
else { \
MiDecrementShareCount (FRAME); \
}
//
// Routines which operate on the Page Frame Database Lists
//
VOID
FASTCALL
MiInsertPageInList (
IN PMMPFNLIST ListHead,
IN PFN_NUMBER PageFrameIndex
);
VOID
FASTCALL
MiInsertPageInFreeList (
IN PFN_NUMBER PageFrameIndex
);
VOID
FASTCALL
MiInsertStandbyListAtFront (
IN PFN_NUMBER PageFrameIndex
);
PFN_NUMBER //PageFrameIndex
FASTCALL
MiRemovePageFromList (
IN PMMPFNLIST ListHead
);
VOID
FASTCALL
MiUnlinkPageFromList (
IN PMMPFN Pfn
);
VOID
MiUnlinkFreeOrZeroedPage (
IN PFN_NUMBER Page
);
VOID
FASTCALL
MiInsertFrontModifiedNoWrite (
IN PFN_NUMBER PageFrameIndex
);
#define MM_MEDIUM_LIMIT 32
#define MM_HIGH_LIMIT 128
ULONG
FASTCALL
MiEnsureAvailablePageOrWait (
IN PEPROCESS Process,
IN PVOID VirtualAddress
);
PFN_NUMBER
MiAllocatePfn (
IN PMMPTE PointerPte,
IN ULONG Protection
);
PFN_NUMBER
FASTCALL
MiRemoveAnyPage (
IN ULONG PageColor
);
PFN_NUMBER
FASTCALL
MiRemoveZeroPage (
IN ULONG PageColor
);
VOID
MiPurgeTransitionList (
VOID
);
PVOID
MiFindContiguousMemory (
IN PFN_NUMBER LowestPfn,
IN PFN_NUMBER HighestPfn,
IN PFN_NUMBER BoundaryPfn,
IN PFN_NUMBER SizeInPages,
IN MEMORY_CACHING_TYPE CacheType,
IN PVOID CallingAddress
);
PVOID
MiCheckForContiguousMemory (
IN PVOID BaseAddress,
IN PFN_NUMBER BaseAddressPages,
IN PFN_NUMBER SizeInPages,
IN PFN_NUMBER LowestPfn,
IN PFN_NUMBER HighestPfn,
IN PFN_NUMBER BoundaryPfn,
IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
);
//
// Routines which operate on the page frame database entry.
//
VOID
MiInitializePfn (
IN PFN_NUMBER PageFrameIndex,
IN PMMPTE PointerPte,
IN ULONG ModifiedState
);
VOID
MiInitializePfnForOtherProcess (
IN PFN_NUMBER PageFrameIndex,
IN PMMPTE PointerPte,
IN PFN_NUMBER ContainingPageFrame
);
VOID
MiInitializeCopyOnWritePfn (
IN PFN_NUMBER PageFrameIndex,
IN PMMPTE PointerPte,
IN WSLE_NUMBER WorkingSetIndex,
IN PVOID SessionSpace
);
VOID
MiInitializeTransitionPfn (
IN PFN_NUMBER PageFrameIndex,
IN PMMPTE PointerPte
);
extern SLIST_HEADER MmInPageSupportSListHead;
VOID
MiFreeInPageSupportBlock (
IN PMMINPAGE_SUPPORT Support
);
PMMINPAGE_SUPPORT
MiGetInPageSupportBlock (
IN LOGICAL PfnHeld,
IN PEPROCESS Process
);
//
// Routines which require a physical page to be mapped into hyperspace
// within the current process.
//
VOID
FASTCALL
MiZeroPhysicalPage (
IN PFN_NUMBER PageFrameIndex,
IN ULONG Color
);
VOID
FASTCALL
MiRestoreTransitionPte (
IN PFN_NUMBER PageFrameIndex
);
PSUBSECTION
MiGetSubsectionAndProtoFromPte (
IN PMMPTE PointerPte,
IN PMMPTE *ProtoPte
);
PVOID
MiMapPageInHyperSpace (
IN PEPROCESS Process,
IN PFN_NUMBER PageFrameIndex,
OUT PKIRQL OldIrql
);
PVOID
MiMapPageInHyperSpaceAtDpc (
IN PEPROCESS Process,
IN PFN_NUMBER PageFrameIndex
);
#define MiUnmapPageInZeroSpace(VA) \
MiGetPteAddress(VA)->u.Long = 0;
PVOID
MiMapImageHeaderInHyperSpace (
IN PFN_NUMBER PageFrameIndex
);
VOID
MiUnmapImageHeaderInHyperSpace (
VOID
);
VOID
MiUpdateImageHeaderPage (
IN PMMPTE PointerPte,
IN PFN_NUMBER PageFrameNumber,
IN PCONTROL_AREA ControlArea
);
PFN_NUMBER
MiGetPageForHeader (
VOID
);
VOID
MiRemoveImageHeaderPage (
IN PFN_NUMBER PageFrameNumber
);
PVOID
MiMapPageToZeroInHyperSpace (
IN PFN_NUMBER PageFrameIndex
);
NTSTATUS
MiGetWritablePagesInSection(
IN PSECTION Section,
OUT PULONG WritablePages
);
//
// Routines to obtain and release system PTEs.
//
PMMPTE
MiReserveSystemPtes (
IN ULONG NumberOfPtes,
IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
);
PMMPTE
MiReserveAlignedSystemPtes (
IN ULONG NumberOfPtes,
IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType,
IN ULONG Alignment
);
VOID
MiReleaseSystemPtes (
IN PMMPTE StartingPte,
IN ULONG NumberOfPtes,
IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
);
VOID
MiReleaseSplitSystemPtes (
IN PMMPTE StartingPte,
IN ULONG NumberOfPtes,
IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
);
VOID
MiIncrementSystemPtes (
IN ULONG NumberOfPtes
);
LOGICAL
MiGetSystemPteAvailability (
IN ULONG NumberOfPtes,
IN MM_PAGE_PRIORITY Priority
);
VOID
MiIssueNoPtesBugcheck (
IN ULONG NumberOfPtes,
IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
);
VOID
MiInitializeSystemPtes (
IN PMMPTE StartingPte,
IN ULONG NumberOfPtes,
IN MMSYSTEM_PTE_POOL_TYPE SystemPteType
);
NTSTATUS
MiAddMappedPtes (
IN PMMPTE FirstPte,
IN ULONG NumberOfPtes,
IN PCONTROL_AREA ControlArea
);
VOID
MiInitializeIoTrackers (
VOID
);
PVOID
MiMapSinglePage (
IN PVOID VirtualAddress OPTIONAL,
IN PFN_NUMBER PageFrameIndex,
IN MEMORY_CACHING_TYPE CacheType,
IN MM_PAGE_PRIORITY Priority
);
VOID
MiUnmapSinglePage (
IN PVOID BaseAddress
);
typedef struct _MM_PTE_MAPPING {
LIST_ENTRY ListEntry;
PVOID SystemVa;
PVOID SystemEndVa;
ULONG Protection;
} MM_PTE_MAPPING, *PMM_PTE_MAPPING;
extern LIST_ENTRY MmProtectedPteList;
extern KSPIN_LOCK MmProtectedPteLock;
LOGICAL
MiCheckSystemPteProtection (
IN ULONG_PTR StoreInstruction,
IN PVOID VirtualAddress
);
//
// Access Fault routines.
//
#define STATUS_ISSUE_PAGING_IO (0xC0033333)
NTSTATUS
MiDispatchFault (
IN ULONG_PTR FaultStatus,
IN PVOID VirtualAdress,
IN PMMPTE PointerPte,
IN PMMPTE PointerProtoPte,
IN PEPROCESS Process,
OUT PLOGICAL ApcNeeded
);
NTSTATUS
MiResolveDemandZeroFault (
IN PVOID VirtualAddress,
IN PMMPTE PointerPte,
IN PEPROCESS Process,
IN ULONG PrototypePte
);
NTSTATUS
MiResolveTransitionFault (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte,
IN PEPROCESS Process,
IN ULONG PfnLockHeld,
OUT PLOGICAL ApcNeeded,
OUT PMMINPAGE_SUPPORT *InPageBlock
);
NTSTATUS
MiResolvePageFileFault (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte,
IN PMMINPAGE_SUPPORT *ReadBlock,
IN PEPROCESS Process
);
NTSTATUS
MiResolveProtoPteFault (
IN ULONG_PTR StoreInstruction,
IN PVOID VirtualAddress,
IN PMMPTE PointerPte,
IN PMMPTE PointerProtoPte,
IN PMMINPAGE_SUPPORT *ReadBlock,
IN PEPROCESS Process,
OUT PLOGICAL ApcNeeded
);
NTSTATUS
MiResolveMappedFileFault (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte,
IN PMMINPAGE_SUPPORT *ReadBlock,
IN PEPROCESS Process
);
VOID
MiAddValidPageToWorkingSet (
IN PVOID VirtualAddress,
IN PMMPTE PointerPte,
IN PMMPFN Pfn1,
IN ULONG WsleMask
);
NTSTATUS
MiWaitForInPageComplete (
IN PMMPFN Pfn,
IN PMMPTE PointerPte,
IN PVOID FaultingAddress,
IN PMMPTE PointerPteContents,
IN PMMINPAGE_SUPPORT InPageSupport,
IN PEPROCESS CurrentProcess
);
LOGICAL
FASTCALL
MiCopyOnWrite (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte
);
VOID
MiSetDirtyBit (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte,
IN ULONG PfnHeld
);
VOID
MiSetModifyBit (
IN PMMPFN Pfn
);
PMMPTE
MiFindActualFaultingPte (
IN PVOID FaultingAddress
);
VOID
MiInitializeReadInProgressSinglePfn (
IN PFN_NUMBER PageFrameIndex,
IN PMMPTE BasePte,
IN PKEVENT Event,
IN WSLE_NUMBER WorkingSetIndex
);
VOID
MiInitializeReadInProgressPfn (
IN PMDL Mdl,
IN PMMPTE BasePte,
IN PKEVENT Event,
IN WSLE_NUMBER WorkingSetIndex
);
NTSTATUS
MiAccessCheck (
IN PMMPTE PointerPte,
IN ULONG_PTR WriteOperation,
IN KPROCESSOR_MODE PreviousMode,
IN ULONG Protection,
IN BOOLEAN CallerHoldsPfnLock
);
NTSTATUS
FASTCALL
MiCheckForUserStackOverflow (
IN PVOID FaultingAddress
);
PMMPTE
MiCheckVirtualAddress (
IN PVOID VirtualAddress,
OUT PULONG ProtectCode
);
NTSTATUS
FASTCALL
MiCheckPdeForPagedPool (
IN PVOID VirtualAddress
);
//
// Routines which operate on an address tree.
//
PMMADDRESS_NODE
FASTCALL
MiGetNextNode (
IN PMMADDRESS_NODE Node
);
PMMADDRESS_NODE
FASTCALL
MiGetPreviousNode (
IN PMMADDRESS_NODE Node
);
PMMADDRESS_NODE
FASTCALL
MiGetFirstNode (
IN PMMADDRESS_NODE Root
);
PMMADDRESS_NODE
MiGetLastNode (
IN PMMADDRESS_NODE Root
);
VOID
FASTCALL
MiInsertNode (
IN PMMADDRESS_NODE Node,
IN OUT PMMADDRESS_NODE *Root
);
VOID
FASTCALL
MiRemoveNode (
IN PMMADDRESS_NODE Node,
IN OUT PMMADDRESS_NODE *Root
);
PMMADDRESS_NODE
FASTCALL
MiLocateAddressInTree (
IN ULONG_PTR Vpn,
IN PMMADDRESS_NODE *Root
);
PMMADDRESS_NODE
MiCheckForConflictingNode (
IN ULONG_PTR StartVpn,
IN ULONG_PTR EndVpn,
IN PMMADDRESS_NODE Root
);
NTSTATUS
MiFindEmptyAddressRangeInTree (
IN SIZE_T SizeOfRange,
IN ULONG_PTR Alignment,
IN PMMADDRESS_NODE Root,
OUT PMMADDRESS_NODE *PreviousVad,
OUT PVOID *Base
);
NTSTATUS
MiFindEmptyAddressRangeDownTree (
IN SIZE_T SizeOfRange,
IN PVOID HighestAddressToEndAt,
IN ULONG_PTR Alignment,
IN PMMADDRESS_NODE Root,
OUT PVOID *Base
);
VOID
NodeTreeWalk (
PMMADDRESS_NODE Start
);
//
// Routines which operate on the tree of virtual address descriptors.
//
NTSTATUS
MiInsertVad (
IN PMMVAD Vad
);
VOID
MiRemoveVad (
IN PMMVAD Vad
);
PMMVAD
FASTCALL
MiLocateAddress (
IN PVOID Vad
);
NTSTATUS
MiFindEmptyAddressRange (
IN SIZE_T SizeOfRange,
IN ULONG_PTR Alignment,
IN ULONG QuickCheck,
IN PVOID *Base
);
//
// Routines which operate on the clone tree structure.
//
NTSTATUS
MiCloneProcessAddressSpace (
IN PEPROCESS ProcessToClone,
IN PEPROCESS ProcessToInitialize,
IN PFN_NUMBER PdePhysicalPage,
IN PFN_NUMBER HyperPhysicalPage
);
ULONG
MiDecrementCloneBlockReference (
IN PMMCLONE_DESCRIPTOR CloneDescriptor,
IN PMMCLONE_BLOCK CloneBlock,
IN PEPROCESS CurrentProcess
);
LOGICAL
MiWaitForForkToComplete (
IN PEPROCESS CurrentProcess,
IN LOGICAL PfnHeld
);
//
// Routines which operate on the working set list.
//
WSLE_NUMBER
MiLocateAndReserveWsle (
IN PMMSUPPORT WsInfo
);
VOID
MiReleaseWsle (
IN WSLE_NUMBER WorkingSetIndex,
IN PMMSUPPORT WsInfo
);
VOID
MiUpdateWsle (
IN PWSLE_NUMBER DesiredIndex,
IN PVOID VirtualAddress,
IN PMMWSL WorkingSetList,
IN PMMPFN Pfn
);
VOID
MiInitializeWorkingSetList (
IN PEPROCESS CurrentProcess
);
VOID
MiGrowWsleHash (
IN PMMSUPPORT WsInfo
);
WSLE_NUMBER
MiTrimWorkingSet (
IN WSLE_NUMBER Reduction,
IN PMMSUPPORT WsInfo,
IN ULONG TrimAge
);
LOGICAL
MmTrimProcessMemory (
IN LOGICAL PurgeTransition
);
LOGICAL
MmTrimSessionMemory (
IN LOGICAL PurgeTransition
);
VOID
MiRemoveWorkingSetPages (
IN PMMWSL WorkingSetList,
IN PMMSUPPORT WsInfo
);
VOID
MiAgeAndEstimateAvailableInWorkingSet (
IN PMMSUPPORT VmSupport,
IN LOGICAL DoAging,
IN PWSLE_NUMBER WslesScanned,
IN OUT PPFN_NUMBER TotalClaim,
IN OUT PPFN_NUMBER TotalEstimatedAvailable
);
VOID
FASTCALL
MiInsertWsleHash (
IN WSLE_NUMBER Entry,
IN PMMWSL WorkingSetList
);
VOID
FASTCALL
MiRemoveWsle (
IN WSLE_NUMBER Entry,
IN PMMWSL WorkingSetList
);
WSLE_NUMBER
FASTCALL
MiLocateWsle (
IN PVOID VirtualAddress,
IN PMMWSL WorkingSetList,
IN WSLE_NUMBER WsPfnIndex
);
ULONG
MiFreeWsle (
IN WSLE_NUMBER WorkingSetIndex,
IN PMMSUPPORT WsInfo,
IN PMMPTE PointerPte
);
VOID
MiSwapWslEntries (
IN WSLE_NUMBER SwapEntry,
IN WSLE_NUMBER Entry,
IN PMMSUPPORT WsInfo
);
VOID
MiRemoveWsleFromFreeList (
IN WSLE_NUMBER Entry,
IN PMMWSLE Wsle,
IN PMMWSL WorkingSetList
);
ULONG
MiRemovePageFromWorkingSet (
IN PMMPTE PointerPte,
IN PMMPFN Pfn1,
IN PMMSUPPORT WsInfo
);
PFN_NUMBER
MiDeleteSystemPagableVm (
IN PMMPTE PointerPte,
IN PFN_NUMBER NumberOfPtes,
IN MMPTE NewPteValue,
IN LOGICAL SessionAllocation,
OUT PPFN_NUMBER ResidentPages OPTIONAL
);
VOID
MiLockCode (
IN PMMPTE FirstPte,
IN PMMPTE LastPte,
IN ULONG LockType
);
PKLDR_DATA_TABLE_ENTRY
MiLookupDataTableEntry (
IN PVOID AddressWithinSection,
IN ULONG ResourceHeld
);
//
// Routines which perform working set management.
//
VOID
MiObtainFreePages (
VOID
);
VOID
MiModifiedPageWriter (
IN PVOID StartContext
);
VOID
MiMappedPageWriter (
IN PVOID StartContext
);
LOGICAL
MiIssuePageExtendRequest (
IN PMMPAGE_FILE_EXPANSION PageExtend
);
VOID
MiIssuePageExtendRequestNoWait (
IN PFN_NUMBER SizeInPages
);
SIZE_T
MiExtendPagingFiles (
IN PMMPAGE_FILE_EXPANSION PageExpand
);
VOID
MiContractPagingFiles (
VOID
);
VOID
MiAttemptPageFileReduction (
VOID
);
LOGICAL
MiCancelWriteOfMappedPfn (
IN PFN_NUMBER PageToStop
);
//
// Routines to delete address space.
//
VOID
MiDeletePteRange (
IN PEPROCESS Process,
IN PMMPTE PointerPte,
IN PMMPTE LastPte,
IN LOGICAL AddressSpaceDeletion
);
VOID
MiDeleteVirtualAddresses (
IN PUCHAR StartingAddress,
IN PUCHAR EndingAddress,
IN ULONG AddressSpaceDeletion,
IN PMMVAD Vad
);
ULONG
MiDeletePte (
IN PMMPTE PointerPte,
IN PVOID VirtualAddress,
IN ULONG AddressSpaceDeletion,
IN PEPROCESS CurrentProcess,
IN PMMPTE PrototypePte,
IN PMMPTE_FLUSH_LIST PteFlushList OPTIONAL
);
VOID
MiDeletePageTablesForPhysicalRange (
IN PVOID StartingAddress,
IN PVOID EndingAddress
);
VOID
MiFlushPteList (
IN PMMPTE_FLUSH_LIST PteFlushList,
IN ULONG AllProcessors,
IN MMPTE FillPte
);
ULONG
FASTCALL
MiReleasePageFileSpace (
IN MMPTE PteContents
);
VOID
FASTCALL
MiReleaseConfirmedPageFileSpace (
IN MMPTE PteContents
);
VOID
FASTCALL
MiUpdateModifiedWriterMdls (
IN ULONG PageFileNumber
);
PVOID
MiAllocateAweInfo (
VOID
);
VOID
MiRemoveUserPhysicalPagesVad (
IN PMMVAD_SHORT FoundVad
);
VOID
MiCleanPhysicalProcessPages (
IN PEPROCESS Process
);
VOID
MiPhysicalViewRemover (
IN PEPROCESS Process,
IN PMMVAD Vad
);
VOID
MiPhysicalViewAdjuster (
IN PEPROCESS Process,
IN PMMVAD OldVad,
IN PMMVAD NewVad
);
LOGICAL
MiIsPhysicalMemoryAddress (
IN PFN_NUMBER PageFrameIndex,
IN OUT PULONG Hint,
IN LOGICAL PfnLockNeeded
);
//
// MM_SYSTEM_PAGE_COLOR - MmSystemPageColor
//
// This variable is updated frequently, on MP systems we keep
// a separate system color per processor to avoid cache line
// thrashing.
//
#if defined(NT_UP)
#define MI_SYSTEM_PAGE_COLOR MmSystemPageColor
#else
#define MI_SYSTEM_PAGE_COLOR (KeGetCurrentPrcb()->PageColor)
#endif
#if defined(MI_MULTINODE)
extern PKNODE KeNodeBlock[];
#define MI_NODE_FROM_COLOR(c) \
(KeNodeBlock[(c) >> MmSecondaryColorNodeShift])
#define MI_GET_COLOR_FROM_LIST_ENTRY(index,pfn) \
((ULONG)(((pfn)->u3.e1.PageColor << MmSecondaryColorNodeShift) | \
MI_GET_SECONDARY_COLOR((index),(pfn))))
#define MI_ADJUST_COLOR_FOR_NODE(c,n) ((c) | (n)->Color)
#define MI_CURRENT_NODE_COLOR (KeGetCurrentNode()->MmShiftedColor)
#define MiRemoveZeroPageIfAny(c) \
(KeGetCurrentNode()->FreeCount[ZeroedPageList] ? MiRemoveZeroPage(c) : 0)
#define MI_GET_PAGE_COLOR_NODE(n) \
(((MI_SYSTEM_PAGE_COLOR++) & MmSecondaryColorMask) | \
KeNodeBlock[n]->MmShiftedColor)
#else
#define MI_NODE_FROM_COLOR(c)
#define MI_GET_COLOR_FROM_LIST_ENTRY(index,pfn) \
((ULONG)MI_GET_SECONDARY_COLOR((index),(pfn)))
#define MI_ADJUST_COLOR_FOR_NODE(c,n) (c)
#define MI_CURRENT_NODE_COLOR 0
#define MiRemoveZeroPageIfAny(COLOR) \
(MmFreePagesByColor[ZeroedPageList][COLOR].Flink != MM_EMPTY_LIST) ? \
MiRemoveZeroPage(COLOR) : 0
#define MI_GET_PAGE_COLOR_NODE(n) \
((MI_SYSTEM_PAGE_COLOR++) & MmSecondaryColorMask)
#endif
FORCEINLINE
PFN_NUMBER
MiRemoveZeroPageMayReleaseLocks (
IN ULONG Color,
IN KIRQL OldIrql
)
/*++
Routine Description:
This routine returns a zeroed page.
It may release and reacquire the PFN lock to do so, as well as mapping
the page in hyperspace to perform the actual zeroing if necessary.
Environment:
Kernel mode. PFN lock held, hyperspace lock NOT held.
--*/
{
PFN_NUMBER PageFrameIndex;
PageFrameIndex = MiRemoveZeroPageIfAny (Color);
if (PageFrameIndex == 0) {
PageFrameIndex = MiRemoveAnyPage (Color);
UNLOCK_PFN (OldIrql);
MiZeroPhysicalPage (PageFrameIndex, Color);
LOCK_PFN (OldIrql);
}
return PageFrameIndex;
}
//
// General support routines.
//
#if (_MI_PAGING_LEVELS <= 3)
//++
//PMMPTE
//MiGetPxeAddress (
// IN PVOID va
// );
//
// Routine Description:
//
// MiGetPxeAddress returns the address of the extended page directory parent
// entry which maps the given virtual address. This is one level above the
// page parent directory.
//
// Arguments
//
// Va - Supplies the virtual address to locate the PXE for.
//
// Return Value:
//
// The address of the PXE.
//
//--
#define MiGetPxeAddress(va) ((PMMPTE)0)
//++
//LOGICAL
//MiIsPteOnPxeBoundary (
// IN PVOID PTE
// );
//
// Routine Description:
//
// MiIsPteOnPxeBoundary returns TRUE if the PTE is
// on an extended page directory parent entry boundary.
//
// Arguments
//
// PTE - Supplies the PTE to check.
//
// Return Value:
//
// TRUE if on a boundary, FALSE if not.
//
//--
#define MiIsPteOnPxeBoundary(PTE) (FALSE)
#endif
#if (_MI_PAGING_LEVELS <= 2)
//++
//PMMPTE
//MiGetPpeAddress (
// IN PVOID va
// );
//
// Routine Description:
//
// MiGetPpeAddress returns the address of the page directory parent entry
// which maps the given virtual address. This is one level above the
// page directory.
//
// Arguments
//
// Va - Supplies the virtual address to locate the PPE for.
//
// Return Value:
//
// The address of the PPE.
//
//--
#define MiGetPpeAddress(va) ((PMMPTE)0)
//++
//LOGICAL
//MiIsPteOnPpeBoundary (
// IN PVOID VA
// );
//
// Routine Description:
//
// MiIsPteOnPpeBoundary returns TRUE if the PTE is
// on a page directory parent entry boundary.
//
// Arguments
//
// VA - Supplies the virtual address to check.
//
// Return Value:
//
// TRUE if on a boundary, FALSE if not.
//
//--
#define MiIsPteOnPpeBoundary(PTE) (FALSE)
#endif
ULONG
MiDoesPdeExistAndMakeValid (
IN PMMPTE PointerPde,
IN PEPROCESS TargetProcess,
IN LOGICAL PfnLockHeld,
OUT PULONG Waited
);
#if (_MI_PAGING_LEVELS >= 3)
#define MiDoesPpeExistAndMakeValid(PPE, PROCESS, PFNLOCKHELD, WAITED) \
MiDoesPdeExistAndMakeValid(PPE, PROCESS, PFNLOCKHELD, WAITED)
#else
#define MiDoesPpeExistAndMakeValid(PPE, PROCESS, PFNLOCKHELD, WAITED) 1
#endif
#if (_MI_PAGING_LEVELS >= 4)
#define MiDoesPxeExistAndMakeValid(PXE, PROCESS, PFNLOCKHELD, WAITED) \
MiDoesPdeExistAndMakeValid(PXE, PROCESS, PFNLOCKHELD, WAITED)
#else
#define MiDoesPxeExistAndMakeValid(PXE, PROCESS, PFNLOCKHELD, WAITED) 1
#endif
VOID
MiMakePdeExistAndMakeValid (
IN PMMPTE PointerPde,
IN PEPROCESS TargetProcess,
IN LOGICAL PfnLockHeld
);
#if (_MI_PAGING_LEVELS >= 4)
VOID
MiMakePxeExistAndMakeValid (
IN PMMPTE PointerPpe,
IN PEPROCESS TargetProcess,
IN LOGICAL PfnLockHeld
);
#else
#define MiMakePxeExistAndMakeValid(PDE, PROCESS, PFNLOCKHELD)
#endif
#if (_MI_PAGING_LEVELS >= 3)
VOID
MiMakePpeExistAndMakeValid (
IN PMMPTE PointerPpe,
IN PEPROCESS TargetProcess,
IN LOGICAL PfnLockHeld
);
#else
#define MiMakePpeExistAndMakeValid(PDE, PROCESS, PFNLOCKHELD)
#endif
ULONG
FASTCALL
MiMakeSystemAddressValid (
IN PVOID VirtualAddress,
IN PEPROCESS CurrentProcess
);
ULONG
FASTCALL
MiMakeSystemAddressValidPfnWs (
IN PVOID VirtualAddress,
IN PEPROCESS CurrentProcess OPTIONAL
);
ULONG
FASTCALL
MiMakeSystemAddressValidPfnSystemWs (
IN PVOID VirtualAddress
);
ULONG
FASTCALL
MiMakeSystemAddressValidPfn (
IN PVOID VirtualAddress
);
ULONG
FASTCALL
MiLockPagedAddress (
IN PVOID VirtualAddress,
IN ULONG PfnLockHeld
);
VOID
FASTCALL
MiUnlockPagedAddress (
IN PVOID VirtualAddress,
IN ULONG PfnLockHeld
);
ULONG
FASTCALL
MiIsPteDecommittedPage (
IN PMMPTE PointerPte
);
ULONG
FASTCALL
MiIsProtectionCompatible (
IN ULONG OldProtect,
IN ULONG NewProtect
);
ULONG
FASTCALL
MiIsPteProtectionCompatible (
IN ULONG OldPteProtection,
IN ULONG NewProtect
);
ULONG
FASTCALL
MiMakeProtectionMask (
IN ULONG Protect
);
ULONG
MiIsEntireRangeCommitted (
IN PVOID StartingAddress,
IN PVOID EndingAddress,
IN PMMVAD Vad,
IN PEPROCESS Process
);
ULONG
MiIsEntireRangeDecommitted (
IN PVOID StartingAddress,
IN PVOID EndingAddress,
IN PMMVAD Vad,
IN PEPROCESS Process
);
LOGICAL
MiCheckProtoPtePageState (
IN PMMPTE PrototypePte,
IN LOGICAL PfnLockHeld,
OUT PLOGICAL DroppedPfnLock
);
//++
//PMMPTE
//MiGetProtoPteAddress (
// IN PMMPTE VAD,
// IN PVOID VA
// );
//
// Routine Description:
//
// MiGetProtoPteAddress returns a pointer to the prototype PTE which
// is mapped by the given virtual address descriptor and address within
// the virtual address descriptor.
//
// Arguments
//
// VAD - Supplies a pointer to the virtual address descriptor that contains
// the VA.
//
// VPN - Supplies the virtual page number.
//
// Return Value:
//
// A pointer to the proto PTE which corresponds to the VA.
//
//--
#define MiGetProtoPteAddress(VAD,VPN) \
((((((VPN) - (VAD)->StartingVpn) << PTE_SHIFT) + \
(ULONG_PTR)(VAD)->FirstPrototypePte) <= (ULONG_PTR)(VAD)->LastContiguousPte) ? \
((PMMPTE)(((((VPN) - (VAD)->StartingVpn) << PTE_SHIFT) + \
(ULONG_PTR)(VAD)->FirstPrototypePte))) : \
MiGetProtoPteAddressExtended ((VAD),(VPN)))
PMMPTE
FASTCALL
MiGetProtoPteAddressExtended (
IN PMMVAD Vad,
IN ULONG_PTR Vpn
);
PSUBSECTION
FASTCALL
MiLocateSubsection (
IN PMMVAD Vad,
IN ULONG_PTR Vpn
);
VOID
MiInitializeSystemCache (
IN ULONG MinimumWorkingSet,
IN ULONG MaximumWorkingSet
);
VOID
MiAdjustWorkingSetManagerParameters(
IN LOGICAL WorkStation
);
VOID
MiNotifyMemoryEvents (
VOID
);
extern PFN_NUMBER MmLowMemoryThreshold;
extern PFN_NUMBER MmHighMemoryThreshold;
//
// Section support
//
VOID
FASTCALL
MiInsertBasedSection (
IN PSECTION Section
);
NTSTATUS
MiMapViewOfPhysicalSection (
IN PCONTROL_AREA ControlArea,
IN PEPROCESS Process,
IN PVOID *CapturedBase,
IN PLARGE_INTEGER SectionOffset,
IN PSIZE_T CapturedViewSize,
IN ULONG ProtectionMask,
IN ULONG_PTR ZeroBits,
IN ULONG AllocationType,
IN LOGICAL WriteCombined
);
NTSTATUS
MiMapViewOfDataSection (
IN PCONTROL_AREA ControlArea,
IN PEPROCESS Process,
IN PVOID *CapturedBase,
IN PLARGE_INTEGER SectionOffset,
IN PSIZE_T CapturedViewSize,
IN PSECTION Section,
IN SECTION_INHERIT InheritDisposition,
IN ULONG ProtectionMask,
IN SIZE_T CommitSize,
IN ULONG_PTR ZeroBits,
IN ULONG AllocationType
);
NTSTATUS
MiUnmapViewOfSection (
IN PEPROCESS Process,
IN PVOID BaseAddress,
IN LOGICAL AddressSpaceMutexHeld
);
VOID
MiRemoveImageSectionObject(
IN PFILE_OBJECT File,
IN PCONTROL_AREA ControlArea
);
VOID
MiAddSystemPtes(
IN PMMPTE StartingPte,
IN ULONG NumberOfPtes,
IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
);
VOID
MiRemoveMappedView (
IN PEPROCESS CurrentProcess,
IN PMMVAD Vad
);
VOID
MiSegmentDelete (
PSEGMENT Segment
);
VOID
MiSectionDelete (
IN PVOID Object
);
VOID
MiDereferenceSegmentThread (
IN PVOID StartContext
);
NTSTATUS
MiCreateImageFileMap (
IN PFILE_OBJECT File,
OUT PSEGMENT *Segment
);
NTSTATUS
MiCreateDataFileMap (
IN PFILE_OBJECT File,
OUT PSEGMENT *Segment,
IN PUINT64 MaximumSize,
IN ULONG SectionPageProtection,
IN ULONG AllocationAttributes,
IN ULONG IgnoreFileSizing
);
NTSTATUS
MiCreatePagingFileMap (
OUT PSEGMENT *Segment,
IN PUINT64 MaximumSize,
IN ULONG ProtectionMask,
IN ULONG AllocationAttributes
);
VOID
MiPurgeSubsectionInternal (
IN PSUBSECTION Subsection,
IN ULONG PteOffset
);
VOID
MiPurgeImageSection (
IN PCONTROL_AREA ControlArea,
IN PEPROCESS Process
);
VOID
MiCleanSection (
IN PCONTROL_AREA ControlArea,
IN LOGICAL DirtyDataPagesOk
);
VOID
MiDereferenceControlArea (
IN PCONTROL_AREA ControlArea
);
VOID
MiCheckControlArea (
IN PCONTROL_AREA ControlArea,
IN PEPROCESS CurrentProcess,
IN KIRQL PreviousIrql
);
LOGICAL
MiCheckPurgeAndUpMapCount (
IN PCONTROL_AREA ControlArea
);
VOID
MiCheckForControlAreaDeletion (
IN PCONTROL_AREA ControlArea
);
LOGICAL
MiCheckControlAreaStatus (
IN SECTION_CHECK_TYPE SectionCheckType,
IN PSECTION_OBJECT_POINTERS SectionObjectPointers,
IN ULONG DelayClose,
OUT PCONTROL_AREA *ControlArea,
OUT PKIRQL OldIrql
);
extern SLIST_HEADER MmEventCountSListHead;
PEVENT_COUNTER
MiGetEventCounter (
VOID
);
VOID
MiFreeEventCounter (
IN PEVENT_COUNTER Support
);
ULONG
MiCanFileBeTruncatedInternal (
IN PSECTION_OBJECT_POINTERS SectionPointer,
IN PLARGE_INTEGER NewFileSize OPTIONAL,
IN LOGICAL BlockNewViews,
OUT PKIRQL PreviousIrql
);
#define STATUS_MAPPED_WRITER_COLLISION (0xC0033333)
NTSTATUS
MiFlushSectionInternal (
IN PMMPTE StartingPte,
IN PMMPTE FinalPte,
IN PSUBSECTION FirstSubsection,
IN PSUBSECTION LastSubsection,
IN ULONG Synchronize,
IN LOGICAL WriteInProgressOk,
OUT PIO_STATUS_BLOCK IoStatus
);
//
// protection stuff...
//
NTSTATUS
MiProtectVirtualMemory (
IN PEPROCESS Process,
IN PVOID *CapturedBase,
IN PSIZE_T CapturedRegionSize,
IN ULONG Protect,
IN PULONG LastProtect
);
ULONG
MiGetPageProtection (
IN PMMPTE PointerPte,
IN PEPROCESS Process,
IN LOGICAL PteCapturedToLocalStack
);
NTSTATUS
MiSetProtectionOnSection (
IN PEPROCESS Process,
IN PMMVAD Vad,
IN PVOID StartingAddress,
IN PVOID EndingAddress,
IN ULONG NewProtect,
OUT PULONG CapturedOldProtect,
IN ULONG DontCharge,
OUT PULONG Locked
);
NTSTATUS
MiCheckSecuredVad (
IN PMMVAD Vad,
IN PVOID Base,
IN ULONG_PTR Size,
IN ULONG ProtectionMask
);
HANDLE
MiSecureVirtualMemory (
IN PVOID Address,
IN SIZE_T Size,
IN ULONG ProbeMode,
IN LOGICAL AddressSpaceMutexHeld
);
VOID
MiUnsecureVirtualMemory (
IN HANDLE SecureHandle,
IN LOGICAL AddressSpaceMutexHeld
);
ULONG
MiChangeNoAccessForkPte (
IN PMMPTE PointerPte,
IN ULONG ProtectionMask
);
VOID
MiSetImageProtect (
IN PSEGMENT Segment,
IN ULONG Protection
);
//
// Routines for charging quota and commitment.
//
VOID
MiTrimSegmentCache (
VOID
);
VOID
MiInitializeCommitment (
VOID
);
LOGICAL
FASTCALL
MiChargeCommitment (
IN SIZE_T QuotaCharge,
IN PEPROCESS Process OPTIONAL
);
LOGICAL
FASTCALL
MiChargeCommitmentCantExpand (
IN SIZE_T QuotaCharge,
IN ULONG MustSucceed
);
LOGICAL
FASTCALL
MiChargeTemporaryCommitmentForReduction (
IN SIZE_T QuotaCharge
);
#if defined (_MI_DEBUG_COMMIT_LEAKS)
VOID
FASTCALL
MiReturnCommitment (
IN SIZE_T QuotaCharge
);
#else
#define MiReturnCommitment(_QuotaCharge) \
ASSERT ((SSIZE_T)(_QuotaCharge) >= 0); \
ASSERT (MmTotalCommittedPages >= (_QuotaCharge)); \
InterlockedExchangeAddSizeT (&MmTotalCommittedPages, 0-((SIZE_T)(_QuotaCharge))); \
MM_TRACK_COMMIT (MM_DBG_COMMIT_RETURN_NORMAL, (_QuotaCharge));
#endif
VOID
MiCauseOverCommitPopup (
VOID
);
extern SIZE_T MmPeakCommitment;
extern SIZE_T MmTotalCommitLimitMaximum;
SIZE_T
MiCalculatePageCommitment (
IN PVOID StartingAddress,
IN PVOID EndingAddress,
IN PMMVAD Vad,
IN PEPROCESS Process
);
VOID
MiReturnPageTablePageCommitment (
IN PVOID StartingAddress,
IN PVOID EndingAddress,
IN PEPROCESS CurrentProcess,
IN PMMVAD PreviousVad,
IN PMMVAD NextVad
);
VOID
MiFlushAllPages (
VOID
);
VOID
MiModifiedPageWriterTimerDispatch (
IN PKDPC Dpc,
IN PVOID DeferredContext,
IN PVOID SystemArgument1,
IN PVOID SystemArgument2
);
LONGLONG
MiStartingOffset(
IN PSUBSECTION Subsection,
IN PMMPTE PteAddress
);
LARGE_INTEGER
MiEndingOffset(
IN PSUBSECTION Subsection
);
VOID
MiReloadBootLoadedDrivers (
IN PLOADER_PARAMETER_BLOCK LoaderBlock
);
LOGICAL
MiInitializeLoadedModuleList (
IN PLOADER_PARAMETER_BLOCK LoaderBlock
);
extern ULONG MmSpecialPoolTag;
extern PVOID MmSpecialPoolStart;
extern PVOID MmSpecialPoolEnd;
extern PVOID MmSessionSpecialPoolStart;
extern PVOID MmSessionSpecialPoolEnd;
LOGICAL
MiInitializeSpecialPool (
IN POOL_TYPE PoolType
);
LOGICAL
MiIsSpecialPoolAddressNonPaged (
IN PVOID VirtualAddress
);
#if defined (_WIN64)
LOGICAL
MiInitializeSessionSpecialPool (
VOID
);
VOID
MiDeleteSessionSpecialPool (
VOID
);
#endif
#if defined (_X86_)
LOGICAL
MiRecoverSpecialPtes (
IN ULONG NumberOfPtes
);
#endif
VOID
MiEnableRandomSpecialPool (
IN LOGICAL Enable
);
LOGICAL
MiTriageSystem (
IN PLOADER_PARAMETER_BLOCK LoaderBlock
);
LOGICAL
MiTriageAddDrivers (
IN PLOADER_PARAMETER_BLOCK LoaderBlock
);
LOGICAL
MiTriageVerifyDriver (
IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
);
extern ULONG MmTriageActionTaken;
#if defined (_WIN64)
#define MM_SPECIAL_POOL_PTES ((1024 * 1024) / sizeof (MMPTE))
#else
#define MM_SPECIAL_POOL_PTES (24 * PTE_PER_PAGE)
#endif
#define MI_SUSPECT_DRIVER_BUFFER_LENGTH 512
extern WCHAR MmVerifyDriverBuffer[];
extern ULONG MmVerifyDriverBufferLength;
extern ULONG MmVerifyDriverLevel;
extern LOGICAL MmDontVerifyRandomDrivers;
extern LOGICAL MmSnapUnloads;
extern LOGICAL MmProtectFreedNonPagedPool;
extern ULONG MmEnforceWriteProtection;
extern LOGICAL MmTrackLockedPages;
extern ULONG MmTrackPtes;
#define VI_POOL_FREELIST_END ((ULONG_PTR)-1)
typedef struct _VI_POOL_ENTRY_INUSE {
PVOID VirtualAddress;
PVOID CallingAddress;
SIZE_T NumberOfBytes;
ULONG_PTR Tag;
} VI_POOL_ENTRY_INUSE, *PVI_POOL_ENTRY_INUSE;
typedef struct _VI_POOL_ENTRY {
union {
VI_POOL_ENTRY_INUSE InUse;
ULONG_PTR FreeListNext;
};
} VI_POOL_ENTRY, *PVI_POOL_ENTRY;
#define MI_VERIFIER_ENTRY_SIGNATURE 0x98761940
typedef struct _MI_VERIFIER_DRIVER_ENTRY {
LIST_ENTRY Links;
ULONG Loads;
ULONG Unloads;
UNICODE_STRING BaseName;
PVOID StartAddress;
PVOID EndAddress;
#define VI_VERIFYING_DIRECTLY 0x1
#define VI_VERIFYING_INVERSELY 0x2
#define VI_DISABLE_VERIFICATION 0x4
ULONG Flags;
ULONG_PTR Signature;
ULONG_PTR Reserved;
KSPIN_LOCK VerifierPoolLock;
PVI_POOL_ENTRY PoolHash;
ULONG_PTR PoolHashSize;
ULONG_PTR PoolHashFree;
ULONG_PTR PoolHashReserved;
ULONG CurrentPagedPoolAllocations;
ULONG CurrentNonPagedPoolAllocations;
ULONG PeakPagedPoolAllocations;
ULONG PeakNonPagedPoolAllocations;
SIZE_T PagedBytes;
SIZE_T NonPagedBytes;
SIZE_T PeakPagedBytes;
SIZE_T PeakNonPagedBytes;
} MI_VERIFIER_DRIVER_ENTRY, *PMI_VERIFIER_DRIVER_ENTRY;
typedef struct _MI_VERIFIER_POOL_HEADER {
ULONG_PTR ListIndex;
PMI_VERIFIER_DRIVER_ENTRY Verifier;
} MI_VERIFIER_POOL_HEADER, *PMI_VERIFIER_POOL_HEADER;
typedef struct _MM_DRIVER_VERIFIER_DATA {
ULONG Level;
ULONG RaiseIrqls;
ULONG AcquireSpinLocks;
ULONG SynchronizeExecutions;
ULONG AllocationsAttempted;
ULONG AllocationsSucceeded;
ULONG AllocationsSucceededSpecialPool;
ULONG AllocationsWithNoTag;
ULONG TrimRequests;
ULONG Trims;
ULONG AllocationsFailed;
ULONG AllocationsFailedDeliberately;
ULONG Loads;
ULONG Unloads;
ULONG UnTrackedPool;
ULONG UserTrims;
ULONG CurrentPagedPoolAllocations;
ULONG CurrentNonPagedPoolAllocations;
ULONG PeakPagedPoolAllocations;
ULONG PeakNonPagedPoolAllocations;
SIZE_T PagedBytes;
SIZE_T NonPagedBytes;
SIZE_T PeakPagedBytes;
SIZE_T PeakNonPagedBytes;
ULONG BurstAllocationsFailedDeliberately;
ULONG SessionTrims;
ULONG Reserved[2];
} MM_DRIVER_VERIFIER_DATA, *PMM_DRIVER_VERIFIER_DATA;
LOGICAL
MiInitializeDriverVerifierList (
IN PLOADER_PARAMETER_BLOCK LoaderBlock
);
LOGICAL
MiInitializeVerifyingComponents (
IN PLOADER_PARAMETER_BLOCK LoaderBlock
);
LOGICAL
MiApplyDriverVerifier (
IN PKLDR_DATA_TABLE_ENTRY,
IN PMI_VERIFIER_DRIVER_ENTRY Verifier
);
VOID
MiReApplyVerifierToLoadedModules(
IN PLIST_ENTRY ModuleListHead
);
VOID
MiVerifyingDriverUnloading (
IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
);
VOID
MiVerifierCheckThunks (
IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
);
extern ULONG MiActiveVerifierThunks;
extern LIST_ENTRY MiSuspectDriverList;
extern ULONG MiVerifierThunksAdded;
VOID
MiEnableKernelVerifier (
VOID
);
extern LOGICAL KernelVerifier;
extern MM_DRIVER_VERIFIER_DATA MmVerifierData;
#define MI_FREED_SPECIAL_POOL_SIGNATURE 0x98764321
#define MI_STACK_BYTES 1024
typedef struct _MI_FREED_SPECIAL_POOL {
POOL_HEADER OverlaidPoolHeader;
MI_VERIFIER_POOL_HEADER OverlaidVerifierPoolHeader;
ULONG Signature;
ULONG TickCount;
ULONG NumberOfBytesRequested;
ULONG Pagable;
PVOID VirtualAddress;
PVOID StackPointer;
ULONG StackBytes;
PETHREAD Thread;
UCHAR StackData[MI_STACK_BYTES];
} MI_FREED_SPECIAL_POOL, *PMI_FREED_SPECIAL_POOL;
#define MM_DBG_COMMIT_NONPAGED_POOL_EXPANSION 0
#define MM_DBG_COMMIT_PAGED_POOL_PAGETABLE 1
#define MM_DBG_COMMIT_PAGED_POOL_PAGES 2
#define MM_DBG_COMMIT_SESSION_POOL_PAGE_TABLES 3
#define MM_DBG_COMMIT_ALLOCVM1 4
#define MM_DBG_COMMIT_ALLOCVM_SEGMENT 5
#define MM_DBG_COMMIT_IMAGE 6
#define MM_DBG_COMMIT_PAGEFILE_BACKED_SHMEM 7
#define MM_DBG_COMMIT_INDEPENDENT_PAGES 8
#define MM_DBG_COMMIT_CONTIGUOUS_PAGES 9
#define MM_DBG_COMMIT_MDL_PAGES 0xA
#define MM_DBG_COMMIT_NONCACHED_PAGES 0xB
#define MM_DBG_COMMIT_MAPVIEW_DATA 0xC
#define MM_DBG_COMMIT_FILL_SYSTEM_DIRECTORY 0xD
#define MM_DBG_COMMIT_EXTRA_SYSTEM_PTES 0xE
#define MM_DBG_COMMIT_DRIVER_PAGING_AT_INIT 0xF
#define MM_DBG_COMMIT_PAGEFILE_FULL 0x10
#define MM_DBG_COMMIT_PROCESS_CREATE 0x11
#define MM_DBG_COMMIT_KERNEL_STACK_CREATE 0x12
#define MM_DBG_COMMIT_SET_PROTECTION 0x13
#define MM_DBG_COMMIT_SESSION_CREATE 0x14
#define MM_DBG_COMMIT_SESSION_IMAGE_PAGES 0x15
#define MM_DBG_COMMIT_SESSION_PAGETABLE_PAGES 0x16
#define MM_DBG_COMMIT_SESSION_SHARED_IMAGE 0x17
#define MM_DBG_COMMIT_DRIVER_PAGES 0x18
#define MM_DBG_COMMIT_INSERT_VAD 0x19
#define MM_DBG_COMMIT_SESSION_WS_INIT 0x1A
#define MM_DBG_COMMIT_SESSION_ADDITIONAL_WS_PAGES 0x1B
#define MM_DBG_COMMIT_SESSION_ADDITIONAL_WS_HASHPAGES 0x1C
#define MM_DBG_COMMIT_SPECIAL_POOL_PAGES 0x1D
#define MM_DBG_COMMIT_SPECIAL_POOL_MAPPING_PAGES 0x1E
#define MM_DBG_COMMIT_SMALL 0x1F
#define MM_DBG_COMMIT_EXTRA_WS_PAGES 0x20
#define MM_DBG_COMMIT_EXTRA_INITIAL_SESSION_WS_PAGES 0x21
#define MM_DBG_COMMIT_ALLOCVM_PROCESS 0x22
#define MM_DBG_COMMIT_INSERT_VAD_PT 0x23
#define MM_DBG_COMMIT_ALLOCVM_PROCESS2 0x24
#define MM_DBG_COMMIT_CHARGE_NORMAL 0x25
#define MM_DBG_COMMIT_CHARGE_CAUSE_POPUP 0x26
#define MM_DBG_COMMIT_CHARGE_CANT_EXPAND 0x27
#define MM_DBG_COMMIT_RETURN_NONPAGED_POOL_EXPANSION 0x40
#define MM_DBG_COMMIT_RETURN_PAGED_POOL_PAGES 0x41
#define MM_DBG_COMMIT_RETURN_SESSION_DATAPAGE 0x42
#define MM_DBG_COMMIT_RETURN_ALLOCVM_SEGMENT 0x43
#define MM_DBG_COMMIT_RETURN_ALLOCVM2 0x44
#define MM_DBG_COMMIT_RETURN_IMAGE_NO_LARGE_CA 0x46
#define MM_DBG_COMMIT_RETURN_PTE_RANGE 0x47
#define MM_DBG_COMMIT_RETURN_NTFREEVM1 0x48
#define MM_DBG_COMMIT_RETURN_NTFREEVM2 0x49
#define MM_DBG_COMMIT_RETURN_INDEPENDENT_PAGES 0x4A
#define MM_DBG_COMMIT_RETURN_AWE_EXCESS 0x4B
#define MM_DBG_COMMIT_RETURN_MDL_PAGES 0x4C
#define MM_DBG_COMMIT_RETURN_NONCACHED_PAGES 0x4D
#define MM_DBG_COMMIT_RETURN_SESSION_CREATE_FAILURE 0x4E
#define MM_DBG_COMMIT_RETURN_PAGETABLES 0x4F
#define MM_DBG_COMMIT_RETURN_PROTECTION 0x50
#define MM_DBG_COMMIT_RETURN_SEGMENT_DELETE1 0x51
#define MM_DBG_COMMIT_RETURN_SEGMENT_DELETE2 0x52
#define MM_DBG_COMMIT_RETURN_PAGEFILE_FULL 0x53
#define MM_DBG_COMMIT_RETURN_SESSION_DEREFERENCE 0x54
#define MM_DBG_COMMIT_RETURN_VAD 0x55
#define MM_DBG_COMMIT_RETURN_PROCESS_CREATE_FAILURE1 0x56
#define MM_DBG_COMMIT_RETURN_PROCESS_DELETE 0x57
#define MM_DBG_COMMIT_RETURN_PROCESS_CLEAN_PAGETABLES 0x58
#define MM_DBG_COMMIT_RETURN_KERNEL_STACK_DELETE 0x59
#define MM_DBG_COMMIT_RETURN_SESSION_DRIVER_LOAD_FAILURE1 0x5A
#define MM_DBG_COMMIT_RETURN_DRIVER_INIT_CODE 0x5B
#define MM_DBG_COMMIT_RETURN_DRIVER_UNLOAD 0x5C
#define MM_DBG_COMMIT_RETURN_DRIVER_UNLOAD1 0x5D
#define MM_DBG_COMMIT_RETURN_NORMAL 0x5E
#define MM_DBG_COMMIT_RETURN_PF_FULL_EXTEND 0x5F
#define MM_DBG_COMMIT_RETURN_EXTENDED 0x60
#define MM_DBG_COMMIT_RETURN_SEGMENT_DELETE3 0x61
#if 0
#define MM_COMMIT_COUNTER_MAX 0x80
#define MM_TRACK_COMMIT(_index, bump) \
if (_index >= MM_COMMIT_COUNTER_MAX) { \
DbgPrint("Mm: Invalid commit counter %d %d\n", _index, MM_COMMIT_COUNTER_MAX); \
DbgBreakPoint(); \
} \
else { \
InterlockedExchangeAddSizeT (&MmTrackCommit[_index], bump); \
}
#define MM_TRACK_COMMIT_REDUCTION(_index, bump) \
if (_index >= MM_COMMIT_COUNTER_MAX) { \
DbgPrint("Mm: Invalid commit counter %d %d\n", _index, MM_COMMIT_COUNTER_MAX); \
DbgBreakPoint(); \
} \
else { \
InterlockedExchangeAddSizeT (&MmTrackCommit[_index], 0 - (bump)); \
}
extern SIZE_T MmTrackCommit[MM_COMMIT_COUNTER_MAX];
#define MI_INCREMENT_TOTAL_PROCESS_COMMIT(_charge) InterlockedExchangeAddSizeT (&MmTotalProcessCommit, (_charge));
#else
#define MM_TRACK_COMMIT(_index, bump)
#define MM_TRACK_COMMIT_REDUCTION(_index, bump)
#define MI_INCREMENT_TOTAL_PROCESS_COMMIT(_charge)
#endif
#define MM_BUMP_COUNTER_MAX 60
extern SIZE_T MmResTrack[MM_BUMP_COUNTER_MAX];
#define MM_BUMP_COUNTER(_index, bump) \
ASSERT (_index < MM_BUMP_COUNTER_MAX); \
InterlockedExchangeAddSizeT (&MmResTrack[_index], (SIZE_T)(bump));
extern ULONG MiSpecialPagesNonPaged;
extern ULONG MiSpecialPagesNonPagedMaximum;
//++
//PFN_NUMBER
//MI_NONPAGABLE_MEMORY_AVAILABLE(
// VOID
// );
//
// Routine Description:
//
// This routine lets callers know how many pages can be charged against
// the resident available, factoring in earlier Mm promises that
// may not have been redeemed at this point (ie: nonpaged pool expansion,
// etc, that must be honored at a later point if requested).
//
// Arguments
//
// None.
//
// Return Value:
//
// The number of currently available pages in the resident available.
//
// N.B. This is a signed quantity and can be negative.
//
//--
#define MI_NONPAGABLE_MEMORY_AVAILABLE() \
((SPFN_NUMBER) \
(MmResidentAvailablePages - \
MmSystemLockPagesCount))
extern ULONG MmLargePageMinimum;
//
// hack stuff for testing.
//
VOID
MiDumpValidAddresses (
VOID
);
VOID
MiDumpPfn ( VOID );
VOID
MiDumpWsl ( VOID );
VOID
MiFormatPte (
IN PMMPTE PointerPte
);
VOID
MiCheckPfn ( VOID );
VOID
MiCheckPte ( VOID );
VOID
MiFormatPfn (
IN PMMPFN PointerPfn
);
extern const MMPTE ZeroPte;
extern const MMPTE ZeroKernelPte;
extern const MMPTE ValidKernelPteLocal;
extern MMPTE ValidKernelPte;
extern MMPTE ValidKernelPde;
extern const MMPTE ValidKernelPdeLocal;
extern const MMPTE ValidUserPte;
extern const MMPTE ValidPtePte;
extern const MMPTE ValidPdePde;
extern MMPTE DemandZeroPde;
extern const MMPTE DemandZeroPte;
extern MMPTE KernelPrototypePte;
extern const MMPTE TransitionPde;
extern MMPTE PrototypePte;
extern const MMPTE NoAccessPte;
extern ULONG_PTR MmSubsectionBase;
extern ULONG_PTR MmSubsectionTopPage;
extern ULONG ExpMultiUserTS;
//
// Virtual alignment for PTEs (machine specific) minimum value is
// 4k maximum value is 64k. The maximum value can be raised by
// changing the MM_PROTO_PTE_ALIGNMENT constant and adding more
// reserved mapping PTEs in hyperspace.
//
//
// Total number of physical pages on the system.
//
extern PFN_COUNT MmNumberOfPhysicalPages;
//
// Lowest physical page number on the system.
//
extern PFN_NUMBER MmLowestPhysicalPage;
//
// Highest physical page number on the system.
//
extern PFN_NUMBER MmHighestPhysicalPage;
//
// Highest possible physical page number in the system.
//
extern PFN_NUMBER MmHighestPossiblePhysicalPage;
#if defined (_WIN64)
#define MI_DTC_MAX_PAGES ((PFN_NUMBER)(((ULONG64)128 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
#define MI_DTC_BOOTED_3GB_MAX_PAGES MI_DTC_MAX_PAGES
#define MI_ADS_MAX_PAGES ((PFN_NUMBER)(((ULONG64)64 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
#define MI_DEFAULT_MAX_PAGES ((PFN_NUMBER)(((ULONG64)16 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
#else
#define MI_DTC_MAX_PAGES ((PFN_NUMBER)(((ULONG64)64 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
#define MI_DTC_BOOTED_3GB_MAX_PAGES ((PFN_NUMBER)(((ULONG64)16 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
#define MI_ADS_MAX_PAGES ((PFN_NUMBER)(((ULONG64)32 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
#define MI_DEFAULT_MAX_PAGES ((PFN_NUMBER)(((ULONG64)4 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
#endif
#define MI_BLADE_MAX_PAGES ((PFN_NUMBER)(((ULONG64)2 * 1024 * 1024 * 1024) >> PAGE_SHIFT))
//
// Total number of available pages on the system. This
// is the sum of the pages on the zeroed, free and standby lists.
//
extern PFN_COUNT MmAvailablePages;
//
// Total number of free pages to base working set trimming on.
//
extern PFN_NUMBER MmMoreThanEnoughFreePages;
//
// Total number physical pages which would be usable if every process
// was at it's minimum working set size. This value is initialized
// at system initialization to MmAvailablePages - MM_FLUID_PHYSICAL_PAGES.
// Everytime a thread is created, the kernel stack is subtracted from
// this and every time a process is created, the minimum working set
// is subtracted from this. If the value would become negative, the
// operation (create process/kernel stack/ adjust working set) fails.
// The PFN LOCK must be owned to manipulate this value.
//
extern SPFN_NUMBER MmResidentAvailablePages;
//
// The total number of pages which would be removed from working sets
// if every working set was at its minimum.
//
extern PFN_NUMBER MmPagesAboveWsMinimum;
//
// The total number of pages which would be removed from working sets
// if every working set above its maximum was at its maximum.
//
extern PFN_NUMBER MmPagesAboveWsMaximum;
//
// If memory is becoming short and MmPagesAboveWsMinimum is
// greater than MmPagesAboveWsThreshold, trim working sets.
//
extern PFN_NUMBER MmPagesAboveWsThreshold;
//
// The number of pages to add to a working set if there are ample
// available pages and the working set is below its maximum.
//
extern PFN_NUMBER MmWorkingSetSizeIncrement;
//
// The number of pages to extend the maximum working set size by
// if the working set at its maximum and there are ample available pages.
extern PFN_NUMBER MmWorkingSetSizeExpansion;
extern ULONG MmPlentyFreePages;
//
// The number of pages required to be freed by working set reduction
// before working set reduction is attempted.
//
extern PFN_NUMBER MmWsAdjustThreshold;
//
// The number of pages available to allow the working set to be
// expanded above its maximum.
//
extern PFN_NUMBER MmWsExpandThreshold;
//
// The total number of pages to reduce by working set trimming.
//
extern PFN_NUMBER MmWsTrimReductionGoal;
extern LONG MiDelayPageFaults;
extern PMMPFN MmPfnDatabase;
extern MMPFNLIST MmZeroedPageListHead;
extern MMPFNLIST MmFreePageListHead;
extern MMPFNLIST MmStandbyPageListHead;
extern MMPFNLIST MmRomPageListHead;
extern MMPFNLIST MmModifiedPageListHead;
extern MMPFNLIST MmModifiedNoWritePageListHead;
extern MMPFNLIST MmBadPageListHead;
extern PMMPFNLIST MmPageLocationList[NUMBER_OF_PAGE_LISTS];
extern MMPFNLIST MmModifiedPageListByColor[MM_MAXIMUM_NUMBER_OF_COLORS];
//
// Mask for isolating secondary color from physical page number.
//
extern ULONG MmSecondaryColorMask;
//
// Mask for isolating node color from combined node and secondary
// color.
//
extern ULONG MmSecondaryColorNodeMask;
//
// Width of MmSecondaryColorMask in bits. In multi node systems,
// the node number is combined with the secondary color to make up
// the page color.
//
extern UCHAR MmSecondaryColorNodeShift;
//
// Event for available pages, set means pages are available.
//
extern KEVENT MmAvailablePagesEvent;
extern KEVENT MmAvailablePagesEventMedium;
extern KEVENT MmAvailablePagesEventHigh;
//
// Event for the zeroing page thread.
//
extern KEVENT MmZeroingPageEvent;
//
// Boolean to indicate if the zeroing page thread is currently
// active. This is set to true when the zeroing page event is
// set and set to false when the zeroing page thread is done
// zeroing all the pages on the free list.
//
extern BOOLEAN MmZeroingPageThreadActive;
//
// Minimum number of free pages before zeroing page thread starts.
//
extern PFN_NUMBER MmMinimumFreePagesToZero;
//
// Global event to synchronize mapped writing with cleaning segments.
//
extern KEVENT MmMappedFileIoComplete;
//
// Hyper space items.
//
extern PMMPTE MmFirstReservedMappingPte;
extern PMMPTE MmLastReservedMappingPte;
//
// System space sizes - MmNonPagedSystemStart to MM_NON_PAGED_SYSTEM_END
// defines the ranges of PDEs which must be copied into a new process's
// address space.
//
extern PVOID MmNonPagedSystemStart;
extern PCHAR MmSystemSpaceViewStart;
extern LOGICAL MmProtectFreedNonPagedPool;
//
// Pool sizes.
//
extern SIZE_T MmSizeOfNonPagedPoolInBytes;
extern SIZE_T MmMinimumNonPagedPoolSize;
extern SIZE_T MmDefaultMaximumNonPagedPool;
extern ULONG MmMaximumNonPagedPoolPercent;
extern ULONG MmMinAdditionNonPagedPoolPerMb;
extern ULONG MmMaxAdditionNonPagedPoolPerMb;
extern SIZE_T MmSizeOfPagedPoolInBytes;
extern SIZE_T MmMaximumNonPagedPoolInBytes;
extern PFN_NUMBER MmAllocatedNonPagedPool;
extern PVOID MmNonPagedPoolExpansionStart;
extern ULONG MmExpandedPoolBitPosition;
extern PFN_NUMBER MmNumberOfFreeNonPagedPool;
extern ULONG MmNumberOfSystemPtes;
extern ULONG MiRequestedSystemPtes;
extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
extern PMMPTE MmSystemPagePtes;
extern ULONG MmSystemPageDirectory[];
extern SIZE_T MmHeapSegmentReserve;
extern SIZE_T MmHeapSegmentCommit;
extern SIZE_T MmHeapDeCommitTotalFreeThreshold;
extern SIZE_T MmHeapDeCommitFreeBlockThreshold;
#define MI_MAX_FREE_LIST_HEADS 4
extern LIST_ENTRY MmNonPagedPoolFreeListHead[MI_MAX_FREE_LIST_HEADS];
//
// Counter for flushes of the entire TB.
//
extern ULONG MmFlushCounter;
//
// Pool start and end.
//
extern PVOID MmNonPagedPoolStart;
extern PVOID MmNonPagedPoolEnd;
extern PVOID MmPagedPoolStart;
extern PVOID MmPagedPoolEnd;
//
// Pool bit maps and other related structures.
//
typedef struct _MM_PAGED_POOL_INFO {
PRTL_BITMAP PagedPoolAllocationMap;
PRTL_BITMAP EndOfPagedPoolBitmap;
PRTL_BITMAP PagedPoolLargeSessionAllocationMap; // HYDRA only
PMMPTE FirstPteForPagedPool;
PMMPTE LastPteForPagedPool;
PMMPTE NextPdeForPagedPoolExpansion;
ULONG PagedPoolHint;
SIZE_T PagedPoolCommit;
SIZE_T AllocatedPagedPool;
} MM_PAGED_POOL_INFO, *PMM_PAGED_POOL_INFO;
extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
extern PVOID MmPageAlignedPoolBase[2];
extern PRTL_BITMAP VerifierLargePagedPoolMap;
//
// MmFirstFreeSystemPte contains the offset from the
// Nonpaged system base to the first free system PTE.
// Note, that an offset of zero indicates an empty list.
//
extern MMPTE MmFirstFreeSystemPte[MaximumPtePoolTypes];
extern ULONG_PTR MiSystemViewStart;
//
// System cache sizes.
//
//extern MMSUPPORT MmSystemCacheWs;
extern PMMWSL MmSystemCacheWorkingSetList;
extern PMMWSLE MmSystemCacheWsle;
extern PVOID MmSystemCacheStart;
extern PVOID MmSystemCacheEnd;
extern PFN_NUMBER MmSystemCacheWsMinimum;
extern PFN_NUMBER MmSystemCacheWsMaximum;
//
// Virtual alignment for PTEs (machine specific) minimum value is
// 0 (no alignment) maximum value is 64k. The maximum value can be raised by
// changing the MM_PROTO_PTE_ALIGNMENT constant and adding more
// reserved mapping PTEs in hyperspace.
//
extern ULONG MmAliasAlignment;
//
// Mask to AND with virtual address to get an offset to go
// with the alignment. This value is page aligned.
//
extern ULONG MmAliasAlignmentOffset;
//
// Mask to and with PTEs to determine if the alias mapping is compatible.
// This value is usually (MmAliasAlignment - 1)
//
extern ULONG MmAliasAlignmentMask;
//
// Cells to track unused thread kernel stacks to avoid TB flushes
// every time a thread terminates.
//
extern ULONG MmMaximumDeadKernelStacks;
extern SLIST_HEADER MmDeadStackSListHead;
//
// MmSystemPteBase contains the address of 1 PTE before
// the first free system PTE (zero indicates an empty list).
// The value of this field does not change once set.
//
extern PMMPTE MmSystemPteBase;
//
// Root of system space virtual address descriptors. These define
// the pagable portion of the system.
//
extern PMMVAD MmVirtualAddressDescriptorRoot;
extern PMMADDRESS_NODE MmSectionBasedRoot;
extern PVOID MmHighSectionBase;
//
// Section commit mutex.
//
extern FAST_MUTEX MmSectionCommitMutex;
//
// Section base address mutex.
//
extern FAST_MUTEX MmSectionBasedMutex;
//
// Resource for section extension.
//
extern ERESOURCE MmSectionExtendResource;
extern ERESOURCE MmSectionExtendSetResource;
//
// Inpage cluster sizes for executable pages (set based on memory size).
//
extern ULONG MmDataClusterSize;
extern ULONG MmCodeClusterSize;
//
// Pagefile creation mutex.
//
extern FAST_MUTEX MmPageFileCreationLock;
//
// Event to set when first paging file is created.
//
extern PKEVENT MmPagingFileCreated;
//
// Paging file debug information.
//
extern ULONG_PTR MmPagingFileDebug[];
//
// Spinlock which guards PFN database. This spinlock is used by
// memory management for accessing the PFN database. The I/O
// system makes use of it for unlocking pages during I/O complete.
//
// extern KSPIN_LOCK MmPfnLock;
//
// Spinlock which guards the working set list for the system shared
// address space (paged pool, system cache, pagable drivers).
//
extern ERESOURCE MmSystemWsLock;
//
// Spin lock for allowing working set expansion.
//
extern KSPIN_LOCK MmExpansionLock;
//
// To prevent optimizations.
//
extern MMPTE GlobalPte;
//
// Page color for system working set.
//
extern ULONG MmSystemPageColor;
extern ULONG MmSecondaryColors;
extern ULONG MmProcessColorSeed;
//
// Set from ntos\config\CMDAT3.C Used by customers to disable paging
// of executive on machines with lots of memory. Worth a few TPS on a
// data base server.
//
#define MM_SYSTEM_CODE_LOCKED_DOWN 0x1
#define MM_PAGED_POOL_LOCKED_DOWN 0x2
extern ULONG MmDisablePagingExecutive;
//
// For debugging.
#if DBG
extern ULONG MmDebug;
#endif
//
// Unused segment management
//
extern MMDEREFERENCE_SEGMENT_HEADER MmDereferenceSegmentHeader;
extern LIST_ENTRY MmUnusedSegmentList;
extern LIST_ENTRY MmUnusedSubsectionList;
extern KEVENT MmUnusedSegmentCleanup;
extern ULONG MmConsumedPoolPercentage;
extern ULONG MmUnusedSegmentCount;
extern ULONG MmUnusedSubsectionCount;
extern ULONG MmUnusedSubsectionCountPeak;
extern SIZE_T MiUnusedSubsectionPagedPool;
extern SIZE_T MiUnusedSubsectionPagedPoolPeak;
#define MI_UNUSED_SUBSECTIONS_COUNT_INSERT(_MappedSubsection) \
MmUnusedSubsectionCount += 1; \
if (MmUnusedSubsectionCount > MmUnusedSubsectionCountPeak) { \
MmUnusedSubsectionCountPeak = MmUnusedSubsectionCount; \
} \
MiUnusedSubsectionPagedPool += EX_REAL_POOL_USAGE((_MappedSubsection->PtesInSubsection + _MappedSubsection->UnusedPtes) * sizeof (MMPTE)); \
if (MiUnusedSubsectionPagedPool > MiUnusedSubsectionPagedPoolPeak) { \
MiUnusedSubsectionPagedPoolPeak = MiUnusedSubsectionPagedPool; \
} \
#define MI_UNUSED_SUBSECTIONS_COUNT_REMOVE(_MappedSubsection) \
MmUnusedSubsectionCount -= 1; \
MiUnusedSubsectionPagedPool -= EX_REAL_POOL_USAGE((_MappedSubsection->PtesInSubsection + _MappedSubsection->UnusedPtes) * sizeof (MMPTE));
#define MI_FILESYSTEM_NONPAGED_POOL_CHARGE 150
#define MI_FILESYSTEM_PAGED_POOL_CHARGE 1024
//++
//LOGICAL
//MI_UNUSED_SEGMENTS_SURPLUS (
// IN PVOID va
// );
//
// Routine Description:
//
// This routine determines whether a surplus of unused
// segments exist. If so, the caller can initiate a trim to free pool.
//
// Arguments
//
// None.
//
// Return Value:
//
// TRUE if unused segment trimming should be initiated, FALSE if not.
//
//--
#define MI_UNUSED_SEGMENTS_SURPLUS() \
(((ULONG)((MmPagedPoolInfo.AllocatedPagedPool * 100) / (MmSizeOfPagedPoolInBytes >> PAGE_SHIFT)) > MmConsumedPoolPercentage) || \
((ULONG)((MmAllocatedNonPagedPool * 100) / (MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT)) > MmConsumedPoolPercentage))
VOID
MiConvertStaticSubsections (
IN PCONTROL_AREA ControlArea
);
//++
//VOID
//MI_INSERT_UNUSED_SEGMENT (
// IN PCONTROL_AREA _ControlArea
// );
//
// Routine Description:
//
// This routine inserts a control area into the unused segment list,
// also managing the associated pool charges.
//
// Arguments
//
// _ControlArea - Supplies the control area to obtain the pool charges from.
//
// Return Value:
//
// None.
//
//--
#define MI_INSERT_UNUSED_SEGMENT(_ControlArea) \
{ \
MM_PFN_LOCK_ASSERT(); \
if ((_ControlArea->u.Flags.Image == 0) && \
(_ControlArea->FilePointer != NULL) && \
(_ControlArea->u.Flags.PhysicalMemory == 0)) { \
MiConvertStaticSubsections(_ControlArea); \
} \
InsertTailList (&MmUnusedSegmentList, &_ControlArea->DereferenceList); \
MmUnusedSegmentCount += 1; \
}
//++
//VOID
//MI_UNUSED_SEGMENTS_REMOVE_CHARGE (
// IN PCONTROL_AREA _ControlArea
// );
//
// Routine Description:
//
// This routine manages pool charges during removals of segments from
// the unused segment list.
//
// Arguments
//
// _ControlArea - Supplies the control area to obtain the pool charges from.
//
// Return Value:
//
// None.
//
//--
#define MI_UNUSED_SEGMENTS_REMOVE_CHARGE(_ControlArea) \
{ \
MM_PFN_LOCK_ASSERT(); \
MmUnusedSegmentCount -= 1; \
}
//
// List heads
//
extern MMWORKING_SET_EXPANSION_HEAD MmWorkingSetExpansionHead;
extern MMPAGE_FILE_EXPANSION MmAttemptForCantExtend;
//
// Paging files
//
extern MMMOD_WRITER_LISTHEAD MmPagingFileHeader;
extern MMMOD_WRITER_LISTHEAD MmMappedFileHeader;
extern PMMPAGING_FILE MmPagingFile[MAX_PAGE_FILES];
extern LIST_ENTRY MmFreePagingSpaceLow;
extern ULONG MmNumberOfActiveMdlEntries;
extern ULONG MmNumberOfPagingFiles;
extern KEVENT MmModifiedPageWriterEvent;
extern KEVENT MmCollidedFlushEvent;
extern KEVENT MmCollidedLockEvent;
//
// Total number of committed pages.
//
extern SIZE_T MmTotalCommittedPages;
extern SIZE_T MmTotalCommitLimit;
extern SIZE_T MmOverCommit;
extern SIZE_T MmSharedCommit;
// #define _MI_DEBUG_DATA 1 // Uncomment this for data logging
#if defined (_MI_DEBUG_DATA)
#define MI_DATA_BACKTRACE_LENGTH 8
typedef struct _MI_DATA_TRACES {
PETHREAD Thread;
PMMPFN Pfn;
PMMPTE PointerPte;
MMPFN PfnData;
ULONG CallerId;
ULONG DataInThePage[2];
PVOID StackTrace [MI_DATA_BACKTRACE_LENGTH];
} MI_DATA_TRACES, *PMI_DATA_TRACES;
extern LONG MiDataIndex;
extern ULONG MiTrackData;
extern PMI_DATA_TRACES MiDataTraces;
VOID
FORCEINLINE
MiSnapData (
IN PMMPFN Pfn,
IN PMMPTE PointerPte,
IN ULONG CallerId
)
{
KIRQL OldIrql;
PVOID Va;
PMI_DATA_TRACES Information;
ULONG Index;
ULONG Hash;
PEPROCESS CurrentProcess;
if (MiDataTraces == NULL) {
return;
}
Index = InterlockedIncrement(&MiDataIndex);
Index &= (MiTrackData - 1);
Information = &MiDataTraces[Index];
Information->Thread = PsGetCurrentThread ();
Information->Pfn = Pfn;
Information->PointerPte = PointerPte;
Information->PfnData = *Pfn;
Information->CallerId = CallerId;
CurrentProcess = PsGetCurrentProcess ();
Va = MiMapPageInHyperSpace (CurrentProcess, Pfn - MmPfnDatabase, &OldIrql);
RtlCopyMemory (&Information->DataInThePage[0],
Va,
sizeof (Information->DataInThePage));
MiUnmapPageInHyperSpace (CurrentProcess, Va, OldIrql);
RtlZeroMemory (&Information->StackTrace[0], MI_DATA_BACKTRACE_LENGTH * sizeof(PVOID)); \
RtlCaptureStackBackTrace (0, MI_DATA_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
}
#define MI_SNAP_DATA(_Pfn, _Pte, _CallerId) MiSnapData(_Pfn, _Pte, _CallerId)
#else
#define MI_SNAP_DATA(_Pfn, _Pte, _CallerId)
#endif
//
// Modified page writer.
//
extern PFN_NUMBER MmMinimumFreePages;
extern PFN_NUMBER MmFreeGoal;
extern PFN_NUMBER MmModifiedPageMaximum;
extern PFN_NUMBER MmModifiedPageMinimum;
extern ULONG MmModifiedWriteClusterSize;
extern ULONG MmMinimumFreeDiskSpace;
extern ULONG MmPageFileExtension;
extern ULONG MmMinimumPageFileReduction;
extern LARGE_INTEGER MiModifiedPageLife;
extern BOOLEAN MiTimerPending;
extern KEVENT MiMappedPagesTooOldEvent;
extern KDPC MiModifiedPageWriterTimerDpc;
extern KTIMER MiModifiedPageWriterTimer;
//
// System process working set sizes.
//
extern PFN_NUMBER MmSystemProcessWorkingSetMin;
extern PFN_NUMBER MmSystemProcessWorkingSetMax;
extern PFN_NUMBER MmMinimumWorkingSetSize;
//
// Support for debugger's mapping physical memory.
//
extern PMMPTE MmDebugPte;
extern PMMPTE MmCrashDumpPte;
extern ULONG MiOverCommitCallCount;
//
// Event tracing routines
//
extern PPAGE_FAULT_NOTIFY_ROUTINE MmPageFaultNotifyRoutine;
extern SIZE_T MmSystemViewSize;
VOID
FASTCALL
MiIdentifyPfn (
IN PMMPFN Pfn1,
OUT PMMPFN_IDENTITY PfnIdentity
);
#if defined (_WIN64)
#define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd64((PLONGLONG)a, b)
#else
#define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd((PLONG)(a), b)
#endif
//
// This is a special value loaded into an EPROCESS pointer to indicate that
// the action underway is for a Hydra session, not really the current process.
// (Any value could be used here that is not a valid system pointer or NULL).
//
#define HYDRA_PROCESS ((PEPROCESS)1)
#define PREFETCH_PROCESS ((PEPROCESS)2)
#define MI_SESSION_SPACE_STRUCT_SIZE MM_ALLOCATION_GRANULARITY
#if defined (_WIN64)
/*++
Virtual memory layout of session space when loaded down from
0x2000.0002.0000.0000 (IA64) or FFFF.F980.0000.0000 (AMD64) :
Note that the sizes of mapped views, paged pool & images are registry tunable.
+------------------------------------+
2000.0002.0000.0000 | |
| win32k.sys & video drivers |
| (16MB) |
| |
+------------------------------------+
2000.0001.FF00.0000 | |
| MM_SESSION_SPACE & Session WSLs |
| (16MB) |
| |
2000.0001.FEFF.0000 +------------------------------------+
| |
| ... |
| |
+------------------------------------+
2000.0001.FE80.0000 | |
| Mapped Views for this session |
| (104MB) |
| |
+------------------------------------+
2000.0001.F800.0000 | |
| Paged Pool for this session |
| (64MB) |
| |
2000.0001.F400.0000 +------------------------------------+
| Special Pool for this session |
| (64MB) |
| |
2000.0000.0000.0000 +------------------------------------+
--*/
#define MI_SESSION_SPACE_WS_SIZE ((ULONG_PTR)(16*1024*1024) - MI_SESSION_SPACE_STRUCT_SIZE)
#define MI_SESSION_DEFAULT_IMAGE_SIZE ((ULONG_PTR)(16*1024*1024))
#define MI_SESSION_DEFAULT_VIEW_SIZE ((ULONG_PTR)(104*1024*1024))
#define MI_SESSION_DEFAULT_POOL_SIZE ((ULONG_PTR)(64*1024*1024))
#define MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE (MM_VA_MAPPED_BY_PPE)
#else
/*++
Virtual memory layout of session space when loaded down from 0xC0000000.
Note that the sizes of mapped views, paged pool and images are registry
tunable on 32-bit systems (if NOT booted /3GB, as 3GB has very limited
address space).
+------------------------------------+
C0000000 | |
| win32k.sys, video drivers and any |
| rebased NT4 printer drivers. |
| |
| (8MB) |
| |
+------------------------------------+
BF800000 | |
| MM_SESSION_SPACE & Session WSLs |
| (4MB) |
| |
+------------------------------------+
BF400000 | |
| Mapped views for this session |
| (20MB by default, but is |
| registry configurable) |
| |
+------------------------------------+
BE000000 | |
| Paged pool for this session |
| (16MB by default, but is |
| registry configurable) |
| |
BD000000 +------------------------------------+
--*/
#define MI_SESSION_SPACE_WS_SIZE (4*1024*1024 - MI_SESSION_SPACE_STRUCT_SIZE)
#define MI_SESSION_DEFAULT_IMAGE_SIZE (8*1024*1024)
#define MI_SESSION_DEFAULT_VIEW_SIZE (20*1024*1024)
#define MI_SESSION_DEFAULT_POOL_SIZE (16*1024*1024)
#define MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE \
(MM_SYSTEM_CACHE_END_EXTRA - MM_KSEG2_BASE)
#endif
#define MI_SESSION_SPACE_DEFAULT_TOTAL_SIZE \
(MI_SESSION_DEFAULT_IMAGE_SIZE + \
MI_SESSION_SPACE_STRUCT_SIZE + \
MI_SESSION_SPACE_WS_SIZE + \
MI_SESSION_DEFAULT_VIEW_SIZE + \
MI_SESSION_DEFAULT_POOL_SIZE)
extern ULONG_PTR MmSessionBase;
extern PMMPTE MiSessionBasePte;
extern PMMPTE MiSessionLastPte;
extern ULONG_PTR MiSessionSpaceWs;
extern ULONG_PTR MiSessionViewStart;
extern SIZE_T MmSessionViewSize;
extern ULONG_PTR MiSessionImageStart;
extern ULONG_PTR MiSessionImageEnd;
extern SIZE_T MmSessionImageSize;
extern PMMPTE MiSessionImagePteStart;
extern PMMPTE MiSessionImagePteEnd;
extern ULONG_PTR MiSessionPoolStart;
extern ULONG_PTR MiSessionPoolEnd;
extern SIZE_T MmSessionPoolSize;
extern ULONG_PTR MiSessionSpaceEnd;
extern ULONG MiSessionSpacePageTables;
//
// The number of page table pages required to map all of session space.
//
#define MI_SESSION_SPACE_MAXIMUM_PAGE_TABLES \
(MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE / MM_VA_MAPPED_BY_PDE)
extern SIZE_T MmSessionSize; // size of the entire session space.
//
// Macros to determine if a given address lies in the specified session range.
//
#define MI_IS_SESSION_IMAGE_ADDRESS(VirtualAddress) \
((PVOID)(VirtualAddress) >= (PVOID)MiSessionImageStart && (PVOID)(VirtualAddress) < (PVOID)(MiSessionImageEnd))
#define MI_IS_SESSION_POOL_ADDRESS(VirtualAddress) \
((PVOID)(VirtualAddress) >= (PVOID)MiSessionPoolStart && (PVOID)(VirtualAddress) < (PVOID)MiSessionPoolEnd)
#define MI_IS_SESSION_ADDRESS(_VirtualAddress) \
((PVOID)(_VirtualAddress) >= (PVOID)MmSessionBase && (PVOID)(_VirtualAddress) < (PVOID)(MiSessionSpaceEnd))
#define MI_IS_SESSION_PTE(_Pte) \
((PMMPTE)(_Pte) >= MiSessionBasePte && (PMMPTE)(_Pte) < MiSessionLastPte)
#define MI_IS_SESSION_IMAGE_PTE(_Pte) \
((PMMPTE)(_Pte) >= MiSessionImagePteStart && (PMMPTE)(_Pte) < MiSessionImagePteEnd)
#define SESSION_GLOBAL(_Session) (_Session->GlobalVirtualAddress)
#define MM_DBG_SESSION_INITIAL_PAGETABLE_ALLOC 0
#define MM_DBG_SESSION_INITIAL_PAGETABLE_FREE_RACE 1
#define MM_DBG_SESSION_INITIAL_PAGE_ALLOC 2
#define MM_DBG_SESSION_INITIAL_PAGE_FREE_FAIL1 3
#define MM_DBG_SESSION_INITIAL_PAGETABLE_FREE_FAIL1 4
#define MM_DBG_SESSION_WS_PAGE_FREE 5
#define MM_DBG_SESSION_PAGETABLE_ALLOC 6
#define MM_DBG_SESSION_SYSMAPPED_PAGES_ALLOC 7
#define MM_DBG_SESSION_WS_PAGETABLE_ALLOC 8
#define MM_DBG_SESSION_PAGEDPOOL_PAGETABLE_ALLOC 9
#define MM_DBG_SESSION_PAGEDPOOL_PAGETABLE_FREE_FAIL1 10
#define MM_DBG_SESSION_WS_PAGE_ALLOC 11
#define MM_DBG_SESSION_WS_PAGE_ALLOC_GROWTH 12
#define MM_DBG_SESSION_INITIAL_PAGE_FREE 13
#define MM_DBG_SESSION_PAGETABLE_FREE 14
#define MM_DBG_SESSION_PAGEDPOOL_PAGETABLE_ALLOC1 15
#define MM_DBG_SESSION_DRIVER_PAGES_LOCKED 16
#define MM_DBG_SESSION_DRIVER_PAGES_UNLOCKED 17
#define MM_DBG_SESSION_WS_HASHPAGE_ALLOC 18
#define MM_DBG_SESSION_SYSMAPPED_PAGES_COMMITTED 19
#define MM_DBG_SESSION_COMMIT_PAGEDPOOL_PAGES 30
#define MM_DBG_SESSION_COMMIT_DELETE_VM_RETURN 31
#define MM_DBG_SESSION_COMMIT_POOL_FREED 32
#define MM_DBG_SESSION_COMMIT_IMAGE_UNLOAD 33
#define MM_DBG_SESSION_COMMIT_IMAGELOAD_FAILED1 34
#define MM_DBG_SESSION_COMMIT_IMAGELOAD_FAILED2 35
#define MM_DBG_SESSION_COMMIT_IMAGELOAD_NOACCESS 36
#define MM_DBG_SESSION_NP_LOCK_CODE1 38
#define MM_DBG_SESSION_NP_LOCK_CODE2 39
#define MM_DBG_SESSION_NP_SESSION_CREATE 40
#define MM_DBG_SESSION_NP_PAGETABLE_ALLOC 41
#define MM_DBG_SESSION_NP_POOL_CREATE 42
#define MM_DBG_SESSION_NP_COMMIT_IMAGE 43
#define MM_DBG_SESSION_NP_COMMIT_IMAGE_PT 44
#define MM_DBG_SESSION_NP_INIT_WS 45
#define MM_DBG_SESSION_NP_WS_GROW 46
#define MM_DBG_SESSION_NP_HASH_GROW 47
#define MM_DBG_SESSION_NP_PAGE_DRIVER 48
#define MM_DBG_SESSION_NP_POOL_CREATE_FAILED 49
#define MM_DBG_SESSION_NP_WS_PAGE_FREE 50
#define MM_DBG_SESSION_NP_SESSION_DESTROY 51
#define MM_DBG_SESSION_NP_SESSION_PTDESTROY 52
#define MM_DBG_SESSION_NP_DELVA 53
#if DBG
#define MM_SESS_COUNTER_MAX 54
#define MM_BUMP_SESS_COUNTER(_index, bump) \
if (_index >= MM_SESS_COUNTER_MAX) { \
DbgPrint("Mm: Invalid bump counter %d %d\n", _index, MM_SESS_COUNTER_MAX); \
DbgBreakPoint(); \
} \
MmSessionSpace->Debug[_index] += (bump);
typedef struct _MM_SESSION_MEMORY_COUNTERS {
SIZE_T NonPagablePages;
SIZE_T CommittedPages;
} MM_SESSION_MEMORY_COUNTERS, *PMM_SESSION_MEMORY_COUNTERS;
#define MM_SESS_MEMORY_COUNTER_MAX 8
#define MM_SNAP_SESS_MEMORY_COUNTERS(_index) \
if (_index >= MM_SESS_MEMORY_COUNTER_MAX) { \
DbgPrint("Mm: Invalid session mem counter %d %d\n", _index, MM_SESS_MEMORY_COUNTER_MAX); \
DbgBreakPoint(); \
} \
else { \
MmSessionSpace->Debug2[_index].NonPagablePages = MmSessionSpace->NonPagablePages; \
MmSessionSpace->Debug2[_index].CommittedPages = MmSessionSpace->CommittedPages; \
}
#else
#define MM_BUMP_SESS_COUNTER(_index, bump)
#define MM_SNAP_SESS_MEMORY_COUNTERS(_index)
#endif
#define MM_SESSION_FAILURE_NO_IDS 0
#define MM_SESSION_FAILURE_NO_COMMIT 1
#define MM_SESSION_FAILURE_NO_RESIDENT 2
#define MM_SESSION_FAILURE_RACE_DETECTED 3
#define MM_SESSION_FAILURE_NO_SYSPTES 4
#define MM_SESSION_FAILURE_NO_PAGED_POOL 5
#define MM_SESSION_FAILURE_NO_NONPAGED_POOL 6
#define MM_SESSION_FAILURE_NO_IMAGE_VA_SPACE 7
#define MM_SESSION_FAILURE_NO_SESSION_PAGED_POOL 8
#define MM_SESSION_FAILURE_NO_AVAILABLE 9
#define MM_SESSION_FAILURE_CAUSES 10
ULONG MmSessionFailureCauses[MM_SESSION_FAILURE_CAUSES];
#define MM_BUMP_SESSION_FAILURES(_index) MmSessionFailureCauses[_index] += 1;
typedef struct _MM_SESSION_SPACE_FLAGS {
ULONG Initialized : 1;
ULONG Filler0 : 1;
ULONG WorkingSetInserted : 1;
ULONG SessionListInserted : 1;
ULONG HasWsLock : 1;
ULONG DeletePending : 1;
ULONG Filler : 26;
} MM_SESSION_SPACE_FLAGS;
//
// The session space data structure - allocated per session and only visible at
// MM_SESSION_SPACE_BASE when in the context of a process from the session.
// This virtual address space is rotated at context switch time when switching
// from a process in session A to a process in session B. This rotation is
// useful for things like providing paged pool per session so many sessions
// won't exhaust the VA space which backs the system global pool.
//
// A kernel PTE is also allocated to double map this page so that global
// pointers can be maintained to provide system access from any process context.
// This is needed for things like mutexes and WSL chains.
//
typedef struct _MM_SESSION_SPACE {
ULONG ReferenceCount;
union {
ULONG LongFlags;
MM_SESSION_SPACE_FLAGS Flags;
} u;
ULONG SessionId;
//
// All the page tables for session space use this as their parent.
// Note that it's not really a page directory - it's really a page
// table page itself (the one used to map this very structure).
//
// This provides a reference to something that won't go away and
// is relevant regardless of which process within the session is current.
//
PFN_NUMBER SessionPageDirectoryIndex;
//
// This is a pointer in global system address space, used to make various
// fields that can be referenced from any process visible from any process
// context. This is for things like mutexes, WSL chains, etc.
//
struct _MM_SESSION_SPACE *GlobalVirtualAddress;
//
// This is the list of the processes in this group that have
// session space entries.
//
LIST_ENTRY ProcessList;
//
// Pool allocation counts - these are always valid.
//
SIZE_T NonPagedPoolBytes;
SIZE_T PagedPoolBytes;
ULONG NonPagedPoolAllocations;
ULONG PagedPoolAllocations;
//
// This is the count of non paged allocations to support this session
// space. This includes the session structure page table and data pages,
// WSL page table and data pages, session pool page table pages and session
// image page table pages. These are all charged against
// MmResidentAvailable.
//
SIZE_T NonPagablePages;
//
// This is the count of pages in this session that have been charged against
// the systemwide commit. This includes all the NonPagablePages plus the
// data pages they typically map.
//
SIZE_T CommittedPages;
LARGE_INTEGER LastProcessSwappedOutTime;
#if (_MI_PAGING_LEVELS >= 3)
//
// The page directory that maps session space is saved here so
// trimmers can attach.
//
MMPTE PageDirectory;
#else
//
// The second level page tables that map session space are shared
// by all processes in the session.
//
PMMPTE PageTables;
#endif
//
// Session space paged pool support.
//
FAST_MUTEX PagedPoolMutex;
//
// Start of session paged pool virtual space.
//
PVOID PagedPoolStart;
//
// Current end of pool virtual space. Can be extended to the
// end of the session space.
//
PVOID PagedPoolEnd;
//
// PTE pointers for pool.
//
PMMPTE PagedPoolBasePde;
MM_PAGED_POOL_INFO PagedPoolInfo;
ULONG Color;
ULONG ProcessOutSwapCount;
//
// This is the list of system images currently valid in
// this session space. This information is in addition
// to the module global information in PsLoadedModuleList.
//
LIST_ENTRY ImageList;
//
// The system PTE self-map entry.
//
PMMPTE GlobalPteEntry;
ULONG CopyOnWriteCount;
ULONG SessionPoolAllocationFailures[4];
//
// The count of "known attachers and the associated event.
//
ULONG AttachCount;
KEVENT AttachEvent;
PEPROCESS LastProcess;
//
// Working set information.
//
MMSUPPORT Vm;
PMMWSLE Wsle;
ERESOURCE WsLock; // owned by WorkingSetLockOwner
//
// This chain is in global system addresses (not session VAs) and can
// be walked from any system context, ie: for WSL trimming.
//
LIST_ENTRY WsListEntry;
//
// Support for mapping system views into session space. Each desktop
// allocates a 3MB heap and the global system view space is only 48M
// total. This would limit us to only 20-30 users - rotating the
// system view space with each session removes this limitation.
//
MMSESSION Session;
//
// This is the driver object entry for WIN32K.SYS
//
// It is not a real driver object, but contained here
// for information such as the DriverUnload routine.
//
DRIVER_OBJECT Win32KDriverObject;
PETHREAD WorkingSetLockOwner;
//
// Pool descriptor for less than 1 page allocations.
//
POOL_DESCRIPTOR PagedPool;
//
// This is generally decremented in process delete (not clean) so that
// the session data page and mapping PTE can finally be freed when this
// reaches zero. smss is the only process that decrements it in other
// places as smss never exits.
//
LONG ProcessReferenceToSession;
LCID LocaleId;
#if defined (_WIN64)
//
// NT64 has enough virtual address space to support per-session special
// pool.
//
PMMPTE SpecialPoolFirstPte;
PMMPTE SpecialPoolLastPte;
PMMPTE NextPdeForSpecialPoolExpansion;
PMMPTE LastPdeForSpecialPoolExpansion;
PFN_NUMBER SpecialPagesInUse;
#endif
#if defined(_IA64_)
REGION_MAP_INFO SessionMapInfo;
PFN_NUMBER PageDirectoryParentPage;
#endif
#if DBG
ULONG Debug[MM_SESS_COUNTER_MAX];
MM_SESSION_MEMORY_COUNTERS Debug2[MM_SESS_MEMORY_COUNTER_MAX];
#endif
} MM_SESSION_SPACE, *PMM_SESSION_SPACE;
extern PMM_SESSION_SPACE MmSessionSpace;
extern ULONG MiSessionCount;
//
// This could be improved to just flush the non-global TB entries.
//
#define MI_FLUSH_SESSION_TB() KeFlushEntireTb (TRUE, TRUE);
//
// The default number of pages for the session working set minimum & maximum.
//
#define MI_SESSION_SPACE_WORKING_SET_MINIMUM 20
#define MI_SESSION_SPACE_WORKING_SET_MAXIMUM 384
NTSTATUS
MiSessionCommitPageTables (
PVOID StartVa,
PVOID EndVa
);
NTSTATUS
MiInitializeAndChargePfn (
OUT PPFN_NUMBER PageFrameIndex,
IN PMMPTE PointerPde,
IN PFN_NUMBER ContainingPageFrame,
IN LOGICAL SessionAllocation
);
VOID
MiSessionPageTableRelease (
IN PFN_NUMBER PageFrameIndex
);
NTSTATUS
MiInitializeSessionPool (
VOID
);
VOID
MiCheckSessionPoolAllocations (
VOID
);
VOID
MiFreeSessionPoolBitMaps (
VOID
);
VOID
MiDetachSession (
VOID
);
VOID
MiAttachSession (
IN PMM_SESSION_SPACE SessionGlobal
);
PVOID
MiAttachToSecureProcessInSession (
IN PRKAPC_STATE ApcState
);
VOID
MiDetachFromSecureProcessInSession (
IN PVOID OpaqueSession,
IN PRKAPC_STATE ApcState
);
VOID
MiReleaseProcessReferenceToSessionDataPage (
PMM_SESSION_SPACE SessionGlobal
);
#define MM_SET_SESSION_RESOURCE_OWNER(_Thread) \
ASSERT (MmSessionSpace->WorkingSetLockOwner == NULL); \
MmSessionSpace->WorkingSetLockOwner = _Thread;
#define MM_CLEAR_SESSION_RESOURCE_OWNER() \
ASSERT (MmSessionSpace->WorkingSetLockOwner == PsGetCurrentThread()); \
MmSessionSpace->WorkingSetLockOwner = NULL;
#define MM_SESSION_SPACE_WS_LOCK_ASSERT() \
ASSERT (MmSessionSpace->WorkingSetLockOwner == PsGetCurrentThread())
#define LOCK_SESSION_SPACE_WS(OLDIRQL,_Thread) \
ASSERT (KeGetCurrentIrql() <= APC_LEVEL); \
KeRaiseIrql(APC_LEVEL,&OLDIRQL); \
ExAcquireResourceExclusiveLite(&MmSessionSpace->WsLock, TRUE); \
MM_SET_SESSION_RESOURCE_OWNER (_Thread);
#define UNLOCK_SESSION_SPACE_WS(OLDIRQL) \
MM_CLEAR_SESSION_RESOURCE_OWNER (); \
ExReleaseResourceLite (&MmSessionSpace->WsLock); \
KeLowerIrql (OLDIRQL); \
ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
extern PMMPTE MiHighestUserPte;
extern PMMPTE MiHighestUserPde;
#if (_MI_PAGING_LEVELS >= 4)
extern PMMPTE MiHighestUserPpe;
extern PMMPTE MiHighestUserPxe;
#endif
NTSTATUS
MiEmptyWorkingSet (
IN PMMSUPPORT WsInfo,
IN LOGICAL WaitOk
);
//++
//ULONG
//MiGetPdeSessionIndex (
// IN PVOID va
// );
//
// Routine Description:
//
// MiGetPdeSessionIndex returns the session structure index for the PDE
// will (or does) map the given virtual address.
//
// Arguments
//
// Va - Supplies the virtual address to locate the PDE index for.
//
// Return Value:
//
// The index of the PDE entry.
//
//--
#define MiGetPdeSessionIndex(va) ((ULONG)(((ULONG_PTR)(va) - (ULONG_PTR)MmSessionBase) >> PDI_SHIFT))
//
// Session space contains the image loader and tracker, virtual
// address allocator, paged pool allocator, system view image mappings,
// and working set for kernel mode virtual addresses that are instanced
// for groups of processes in a Session process group. This
// process group is identified by a SessionId.
//
// Each Session process group's loaded kernel modules, paged pool
// allocations, working set, and mapped system views are separate from
// other Session process groups, even though they have the same
// virtual addresses.
//
// This is to support the Hydra multi-user Windows NT system by
// replicating WIN32K.SYS, and its complement of video and printer drivers,
// desktop heaps, memory allocations, etc.
//
//
// Structure linked into a session space structure to describe
// which system images in PsLoadedModuleTable and
// SESSION_DRIVER_GLOBAL_LOAD_ADDRESS's
// have been allocated for the current session space.
//
// The reference count tracks the number of loads of this image within
// this session.
//
typedef struct _IMAGE_ENTRY_IN_SESSION {
LIST_ENTRY Link;
PVOID Address;
PVOID LastAddress;
ULONG ImageCountInThisSession;
PMMPTE PrototypePtes;
PKLDR_DATA_TABLE_ENTRY DataTableEntry;
} IMAGE_ENTRY_IN_SESSION, *PIMAGE_ENTRY_IN_SESSION;
extern LIST_ENTRY MiSessionWsList;
NTSTATUS
FASTCALL
MiCheckPdeForSessionSpace(
IN PVOID VirtualAddress
);
NTSTATUS
MiShareSessionImage (
IN PSECTION Section,
IN OUT PSIZE_T ViewSize
);
VOID
MiSessionWideInitializeAddresses (
VOID
);
NTSTATUS
MiSessionWideReserveImageAddress (
IN PUNICODE_STRING pImageName,
IN PSECTION Section,
IN ULONG_PTR Alignment,
OUT PVOID *ppAddr,
OUT PLOGICAL pAlreadyLoaded
);
VOID
MiInitializeSessionIds (
VOID
);
VOID
MiInitializeSessionWsSupport(
VOID
);
VOID
MiSessionAddProcess (
IN PEPROCESS NewProcess
);
VOID
MiSessionRemoveProcess (
VOID
);
NTSTATUS
MiRemovePsLoadedModule(
PKLDR_DATA_TABLE_ENTRY DataTableEntry
);
NTSTATUS
MiRemoveImageSessionWide(
IN PVOID BaseAddr
);
NTSTATUS
MiDeleteSessionVirtualAddresses(
IN PVOID VirtualAddress,
IN SIZE_T NumberOfBytes
);
NTSTATUS
MiUnloadSessionImageByForce (
IN SIZE_T NumberOfPtes,
IN PVOID ImageBase
);
NTSTATUS
MiSessionWideGetImageSize(
IN PVOID BaseAddress,
OUT PSIZE_T NumberOfBytes OPTIONAL,
OUT PSIZE_T CommitPages OPTIONAL
);
PIMAGE_ENTRY_IN_SESSION
MiSessionLookupImage (
IN PVOID BaseAddress
);
NTSTATUS
MiSessionCommitImagePages(
PVOID BaseAddr,
SIZE_T Size
);
VOID
MiSessionUnloadAllImages (
VOID
);
VOID
MiFreeSessionSpaceMap (
VOID
);
NTSTATUS
MiSessionInitializeWorkingSetList (
VOID
);
VOID
MiSessionUnlinkWorkingSet (
VOID
);
NTSTATUS
MiSessionCopyOnWrite (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte
);
VOID
MiSessionOutSwapProcess (
IN PEPROCESS Process
);
VOID
MiSessionInSwapProcess (
IN PEPROCESS Process
);
#if !defined (_X86PAE_)
#define MI_GET_DIRECTORY_FRAME_FROM_PROCESS(_Process) \
MI_GET_PAGE_FRAME_FROM_PTE((PMMPTE)(&((_Process)->Pcb.DirectoryTableBase[0])))
#define MI_GET_HYPER_PAGE_TABLE_FRAME_FROM_PROCESS(_Process) \
MI_GET_PAGE_FRAME_FROM_PTE((PMMPTE)(&((_Process)->Pcb.DirectoryTableBase[1])))
#else
#define MI_GET_DIRECTORY_FRAME_FROM_PROCESS(_Process) \
(MI_GET_PAGE_FRAME_FROM_PTE(((PMMPTE)((_Process)->PaeTop)) + PD_PER_SYSTEM - 1))
#define MI_GET_HYPER_PAGE_TABLE_FRAME_FROM_PROCESS(_Process) \
((PFN_NUMBER)((_Process)->Pcb.DirectoryTableBase[1]))
#endif
#if defined(_MIALT4K_)
NTSTATUS
MiSetCopyPagesFor4kPage (
IN PEPROCESS Process,
IN PMMVAD Vad,
IN OUT PVOID StartingAddress,
IN OUT PVOID EndingAddress,
IN ULONG ProtectionMask,
OUT PMMVAD *CallerNewVad
);
VOID
MiRemoveAliasedVads (
IN PEPROCESS Process,
IN PMMVAD Vad
);
PVOID
MiDuplicateAliasVadList (
IN PMMVAD Vad
);
#endif
//
// The LDR_DATA_TABLE_ENTRY->LoadedImports is used as a list of imported DLLs.
//
// This field is zero if the module was loaded at boot time and the
// import information was never filled in.
//
// This field is -1 if no imports are defined by the module.
//
// This field contains a valid paged pool PLDR_DATA_TABLE_ENTRY pointer
// with a low-order (bit 0) tag of 1 if there is only 1 usable import needed
// by this driver.
//
// This field will contain a valid paged pool PLOAD_IMPORTS pointer in all
// other cases (ie: where at least 2 imports exist).
//
typedef struct _LOAD_IMPORTS {
SIZE_T Count;
PKLDR_DATA_TABLE_ENTRY Entry[1];
} LOAD_IMPORTS, *PLOAD_IMPORTS;
#define LOADED_AT_BOOT ((PLOAD_IMPORTS)0)
#define NO_IMPORTS_USED ((PLOAD_IMPORTS)-2)
#define SINGLE_ENTRY(ImportVoid) ((ULONG)((ULONG_PTR)(ImportVoid) & 0x1))
#define SINGLE_ENTRY_TO_POINTER(ImportVoid) ((PKLDR_DATA_TABLE_ENTRY)((ULONG_PTR)(ImportVoid) & ~0x1))
#define POINTER_TO_SINGLE_ENTRY(Pointer) ((PKLDR_DATA_TABLE_ENTRY)((ULONG_PTR)(Pointer) | 0x1))
//
// This tracks allocated group virtual addresses. The term SESSIONWIDE is used
// to denote data that is the same across all sessions (as opposed to
// per-session data which can vary from session to session).
//
// Since each driver loaded into a session space is linked and fixed up
// against the system image, it must remain at the same virtual address
// across the system regardless of the session.
//
// A list is maintained by the group allocator of which virtual
// addresses are in use and by which DLL.
//
// The reference count tracks the number of sessions that have loaded
// this image.
//
// Access to this structure is guarded by the MmSystemLoadLock.
//
typedef struct _SESSIONWIDE_DRIVER_ADDRESS {
LIST_ENTRY Link;
ULONG ReferenceCount;
PVOID Address;
ULONG_PTR Size;
ULONG_PTR WritablePages;
UNICODE_STRING FullDllName;
} SESSIONWIDE_DRIVER_ADDRESS, *PSESSIONWIDE_DRIVER_ADDRESS;
// #define _MI_DEBUG_RONLY 1 // Uncomment this for session readonly tracking
#if _MI_DEBUG_RONLY
VOID
MiAssertNotSessionData (
IN PMMPTE PointerPte
);
VOID
MiLogSessionDataStart (
IN PKLDR_DATA_TABLE_ENTRY DataTableEntry
);
#define MI_ASSERT_NOT_SESSION_DATA(PTE) MiAssertNotSessionData(PTE)
#define MI_LOG_SESSION_DATA_START(DataTableEntry) MiLogSessionDataStart(DataTableEntry)
#else
#define MI_ASSERT_NOT_SESSION_DATA(PTE)
#define MI_LOG_SESSION_DATA_START(DataTableEntry)
#endif
//
// This tracks driver-specified individual verifier thunks.
//
typedef struct _DRIVER_SPECIFIED_VERIFIER_THUNKS {
LIST_ENTRY ListEntry;
PKLDR_DATA_TABLE_ENTRY DataTableEntry;
ULONG NumberOfThunks;
} DRIVER_SPECIFIED_VERIFIER_THUNKS, *PDRIVER_SPECIFIED_VERIFIER_THUNKS;
// #define _MI_DEBUG_SUB 1 // Uncomment this for subsection logging
#if defined (_MI_DEBUG_SUB)
extern ULONG MiTrackSubs;
#define MI_SUB_BACKTRACE_LENGTH 8
typedef struct _MI_SUB_TRACES {
PETHREAD Thread;
PMSUBSECTION Subsection;
PCONTROL_AREA ControlArea;
ULONG_PTR CallerId;
PVOID StackTrace [MI_SUB_BACKTRACE_LENGTH];
MSUBSECTION SubsectionContents;
CONTROL_AREA ControlAreaContents;
} MI_SUB_TRACES, *PMI_SUB_TRACES;
extern LONG MiSubsectionIndex;
extern PMI_SUB_TRACES MiSubsectionTraces;
VOID
FORCEINLINE
MiSnapSubsection (
IN PMSUBSECTION Subsection,
IN ULONG CallerId
)
{
PMI_SUB_TRACES Information;
PCONTROL_AREA ControlArea;
ULONG Index;
ULONG Hash;
if (MiSubsectionTraces == NULL) {
return;
}
ControlArea = Subsection->ControlArea;
#if 0
if (ControlArea->u.Flags.Mft == 0) {
return;
}
#endif
Index = InterlockedIncrement(&MiSubsectionIndex);
Index &= (MiTrackSubs - 1);
Information = &MiSubsectionTraces[Index];
Information->Subsection = Subsection;
Information->ControlArea = ControlArea;
*(PMSUBSECTION)&Information->SubsectionContents = *Subsection;
*(PCONTROL_AREA)&Information->ControlAreaContents = *ControlArea;
Information->Thread = PsGetCurrentThread();
Information->CallerId = CallerId;
RtlCaptureStackBackTrace (0, MI_SUB_BACKTRACE_LENGTH, Information->StackTrace, &Hash);
}
#define MI_SNAP_SUB(_Sub, callerid) MiSnapSubsection(_Sub, callerid)
#else
#define MI_SNAP_SUB(_Sub, callerid)
#endif
#ifdef _MI_MESSAGE_SERVER
LOGICAL
MiQueueMessage (
IN PVOID Message
);
PVOID
MiRemoveMessage (
VOID
);
#define MI_INSTRUMENT_QUEUE(Message) MiQueueMessage (Message)
#define MI_INSTRUMENTR_QUEUE() MiRemoveMessage ()
#else
#define MI_INSTRUMENT_QUEUE(Message)
#define MI_INSTRUMENTR_QUEUE()
#endif
#endif // MI
Reference in a new issue View git blame Copy permalink