/*++ Copyright (c) 1989 Microsoft Corporation Module Name: mminit.c Abstract: This module contains the initialization for the memory management system. Author: Lou Perazzoli (loup) 20-Mar-1989 Landy Wang (landyw) 02-Jun-1997 Revision History: --*/ #include "mi.h" PMMPTE MmSharedUserDataPte; extern ULONG_PTR MmSystemPtesStart[MaximumPtePoolTypes]; extern PMMPTE MiSpecialPoolFirstPte; extern ULONG MmPagedPoolCommit; extern ULONG MmInPageSupportMinimum; extern PFN_NUMBER MiExpansionPoolPagesInitialCharge; extern ULONG MmAllocationPreference; extern PVOID BBTBuffer; extern PFN_COUNT BBTPagesToReserve; ULONG_PTR MmSubsectionBase; ULONG_PTR MmSubsectionTopPage; ULONG MmDataClusterSize; ULONG MmCodeClusterSize; PFN_NUMBER MmResidentAvailableAtInit; PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock; LIST_ENTRY MmLockConflictList; LIST_ENTRY MmProtectedPteList; KSPIN_LOCK MmProtectedPteLock; LOGICAL MmPagedPoolMaximumDesired = FALSE; #if defined (_MI_DEBUG_SUB) ULONG MiTrackSubs = 0x2000; // Set to nonzero to enable subsection tracking code. LONG MiSubsectionIndex; PMI_SUB_TRACES MiSubsectionTraces; #endif #if defined (_MI_DEBUG_DIRTY) ULONG MiTrackDirtys = 0x10000; // Set to nonzero to enable subsection tracking code. LONG MiDirtyIndex; PMI_DIRTY_TRACES MiDirtyTraces; #endif #if defined (_MI_DEBUG_DATA) ULONG MiTrackData = 0x10000; // Set to nonzero to enable data tracking code. LONG MiDataIndex; PMI_DATA_TRACES MiDataTraces; #endif VOID MiMapBBTMemory ( IN PLOADER_PARAMETER_BLOCK LoaderBlock ); VOID MiEnablePagingTheExecutive( VOID ); VOID MiEnablePagingOfDriverAtInit ( IN PMMPTE PointerPte, IN PMMPTE LastPte ); VOID MiBuildPagedPool ( ); VOID MiWriteProtectSystemImage ( IN PVOID DllBase ); VOID MiInitializePfnTracing ( VOID ); PFN_NUMBER MiPagesInLoaderBlock ( IN PLOADER_PARAMETER_BLOCK LoaderBlock, IN PBOOLEAN IncludeType ); #ifndef NO_POOL_CHECKS VOID MiInitializeSpecialPoolCriteria ( IN VOID ); #endif #ifdef _MI_MESSAGE_SERVER VOID MiInitializeMessageQueue ( VOID ); #endif static VOID MiMemoryLicense ( IN PLOADER_PARAMETER_BLOCK LoaderBlock ); VOID MiInitializeCacheOverrides ( VOID ); // // The thresholds can be overridden by the registry. // PFN_NUMBER MmLowMemoryThreshold; PFN_NUMBER MmHighMemoryThreshold; PKEVENT MiLowMemoryEvent; PKEVENT MiHighMemoryEvent; NTSTATUS MiCreateMemoryEvent ( IN PUNICODE_STRING EventName, OUT PKEVENT *Event ); LOGICAL MiInitializeMemoryEvents ( VOID ); #ifdef ALLOC_PRAGMA #pragma alloc_text(INIT,MmInitSystem) #pragma alloc_text(INIT,MiMapBBTMemory) #pragma alloc_text(INIT,MmInitializeMemoryLimits) #pragma alloc_text(INIT,MmFreeLoaderBlock) #pragma alloc_text(INIT,MiBuildPagedPool) #pragma alloc_text(INIT,MiFindInitializationCode) #pragma alloc_text(INIT,MiEnablePagingTheExecutive) #pragma alloc_text(INIT,MiEnablePagingOfDriverAtInit) #pragma alloc_text(INIT,MiPagesInLoaderBlock) #pragma alloc_text(INIT,MiCreateMemoryEvent) #pragma alloc_text(INIT,MiInitializeMemoryEvents) #pragma alloc_text(INIT,MiInitializeCacheOverrides) #pragma alloc_text(INIT,MiMemoryLicense) #pragma alloc_text(PAGELK,MiFreeInitializationCode) #endif // // Default is a 300 second life span for modified mapped pages - // This can be overridden in the registry. // #ifdef ALLOC_DATA_PRAGMA #pragma data_seg("INITDATA") #endif ULONG MmModifiedPageLifeInSeconds = 300; #ifdef ALLOC_DATA_PRAGMA #pragma data_seg() #endif LARGE_INTEGER MiModifiedPageLife; BOOLEAN MiTimerPending = FALSE; KEVENT MiMappedPagesTooOldEvent; KDPC MiModifiedPageWriterTimerDpc; KTIMER MiModifiedPageWriterTimer; // // The following constants are based on the number PAGES not the // memory size. For convenience the number of pages is calculated // based on a 4k page size. Hence 12mb with 4k page is 3072. // #define MM_SMALL_SYSTEM ((13*1024*1024) / 4096) #define MM_MEDIUM_SYSTEM ((19*1024*1024) / 4096) #define MM_MIN_INITIAL_PAGED_POOL ((32*1024*1024) >> PAGE_SHIFT) #define MM_DEFAULT_IO_LOCK_LIMIT (2 * 1024 * 1024) extern WSLE_NUMBER MmMaximumWorkingSetSize; extern ULONG MmEnforceWriteProtection; extern CHAR MiPteStr[]; extern LONG MiTrimInProgressCount; #if (_MI_PAGING_LEVELS < 3) PFN_NUMBER MmSystemPageDirectory[PD_PER_SYSTEM]; PMMPTE MmSystemPagePtes; #endif ULONG MmTotalSystemCodePages; MM_SYSTEMSIZE MmSystemSize; ULONG MmLargeSystemCache; ULONG MmProductType; extern ULONG MiVerifyAllDrivers; LIST_ENTRY MmLoadedUserImageList; PPAGE_FAULT_NOTIFY_ROUTINE MmPageFaultNotifyRoutine; #if defined (_WIN64) #define MM_ALLOCATION_FRAGMENT (64 * 1024 * 1024) #else #define MM_ALLOCATION_FRAGMENT (64 * 1024) #endif // // Registry-settable. // SIZE_T MmAllocationFragment; #if defined(MI_MULTINODE) HALNUMAPAGETONODE MiNonNumaPageToNodeColor ( IN PFN_NUMBER PageFrameIndex ) /*++ Routine Description: Return the node color of the page. Arguments: PageFrameIndex - Supplies the physical page number. Return Value: Node color is always zero in non-NUMA configurations. --*/ { UNREFERENCED_PARAMETER (PageFrameIndex); return 0; } // // This node determination function pointer is initialized to return 0. // // Architecture-dependent initialization may repoint it to a HAL routine // for NUMA configurations. // PHALNUMAPAGETONODE MmPageToNode = MiNonNumaPageToNodeColor; VOID MiDetermineNode ( IN PFN_NUMBER PageFrameIndex, IN PMMPFN Pfn ) /*++ Routine Description: This routine is called during initial freelist population or when physical memory is being hot-added. It then determines which node (in a multinode NUMA system) the physical memory resides in, and marks the PFN entry accordingly. N.B. The actual page to node determination is machine dependent and done by a routine in the chipset driver or the HAL, called via the MmPageToNode function pointer. Arguments: PageFrameIndex - Supplies the physical page number. Pfn - Supplies a pointer to the PFN database element. Return Value: None. Environment: None although typically this routine is called with the PFN database locked. --*/ { ULONG Temp; ASSERT (Pfn == MI_PFN_ELEMENT(PageFrameIndex)); Temp = MmPageToNode (PageFrameIndex); ASSERT (Temp < MAXIMUM_CCNUMA_NODES); Pfn->u3.e1.PageColor = Temp; } #endif BOOLEAN MmInitSystem ( IN ULONG Phase, IN PLOADER_PARAMETER_BLOCK LoaderBlock ) /*++ Routine Description: This function is called during Phase 0, phase 1 and at the end of phase 1 ("phase 2") initialization. Phase 0 initializes the memory management paging functions, nonpaged and paged pool, the PFN database, etc. Phase 1 initializes the section objects, the physical memory object, and starts the memory management system threads. Phase 2 frees memory used by the OsLoader. Arguments: Phase - System initialization phase. LoaderBlock - Supplies a pointer to the system loader block. Return Value: Returns TRUE if the initialization was successful. Environment: Kernel Mode Only. System initialization. --*/ { PEPROCESS Process; PSINGLE_LIST_ENTRY SingleListEntry; PFN_NUMBER NumberOfPages; HANDLE ThreadHandle; OBJECT_ATTRIBUTES ObjectAttributes; PMMPTE PointerPte; PMMPTE PointerPde; PMMPTE StartPde; PMMPTE EndPde; PMMPFN Pfn1; PFN_NUMBER i, j; PFN_NUMBER DeferredMdlEntries; PFN_NUMBER PageFrameIndex; PFN_NUMBER DirectoryFrameIndex; MMPTE TempPte; KIRQL OldIrql; PLIST_ENTRY NextEntry; PKLDR_DATA_TABLE_ENTRY DataTableEntry; ULONG MaximumSystemCacheSize; ULONG MaximumSystemCacheSizeTotal; PIMAGE_NT_HEADERS NtHeaders; ULONG_PTR SystemPteMultiplier; ULONG_PTR DefaultSystemViewSize; ULONG_PTR SessionEnd; SIZE_T SystemViewMax; SIZE_T HydraImageMax; SIZE_T HydraViewMax; SIZE_T HydraPoolMax; SIZE_T HydraSpaceUsedForSystemViews; BOOLEAN IncludeType[LoaderMaximum]; LOGICAL AutosizingFragment; ULONG VerifierFlags; #if DBG MMPTE Pointer; #endif #if (_MI_PAGING_LEVELS >= 3) LOGICAL FirstPpe; PMMPTE StartPpe; #endif #if (_MI_PAGING_LEVELS >= 4) LOGICAL FirstPxe; PMMPTE StartPxe; #endif #if defined(_X86_) PCHAR ReducedUserVaOption; ULONG UserVaLimit; ULONG ReductionInBytes; #endif j = 0; PointerPde = NULL; // // Make sure structure alignment is okay. // if (Phase == 0) { MmThrottleTop = 450; MmThrottleBottom = 127; // // Set the highest user address, the system range start address, the // user probe address, and the virtual bias. // #if defined(_WIN64) MmHighestUserAddress = MI_HIGHEST_USER_ADDRESS; MmUserProbeAddress = MI_USER_PROBE_ADDRESS; MmSystemRangeStart = MI_SYSTEM_RANGE_START; #else MmHighestUserAddress = (PVOID)(KSEG0_BASE - 0x10000 - 1); MmUserProbeAddress = KSEG0_BASE - 0x10000; MmSystemRangeStart = (PVOID)KSEG0_BASE; #endif MiHighestUserPte = MiGetPteAddress (MmHighestUserAddress); MiHighestUserPde = MiGetPdeAddress (MmHighestUserAddress); #if (_MI_PAGING_LEVELS >= 4) MiHighestUserPpe = MiGetPpeAddress (MmHighestUserAddress); MiHighestUserPxe = MiGetPxeAddress (MmHighestUserAddress); #endif #if defined(_X86_) || defined(_AMD64_) MmBootImageSize = LoaderBlock->Extension->LoaderPagesSpanned; MmBootImageSize *= PAGE_SIZE; MmBootImageSize = MI_ROUND_TO_SIZE (MmBootImageSize, MM_VA_MAPPED_BY_PDE); ASSERT ((MmBootImageSize % MM_VA_MAPPED_BY_PDE) == 0); #endif #if defined(_X86_) MmVirtualBias = LoaderBlock->u.I386.VirtualBias; #endif // // Initialize system and Hydra mapped view sizes. // DefaultSystemViewSize = MM_SYSTEM_VIEW_SIZE; MmSessionSize = MI_SESSION_SPACE_DEFAULT_TOTAL_SIZE; SessionEnd = (ULONG_PTR) MM_SESSION_SPACE_DEFAULT_END; #define MM_MB_MAPPED_BY_PDE (MM_VA_MAPPED_BY_PDE / (1024*1024)) // // A PDE of virtual space is the minimum system view size allowed. // if (MmSystemViewSize < (MM_VA_MAPPED_BY_PDE / (1024*1024))) { MmSystemViewSize = DefaultSystemViewSize; } else { // // The view size has been specified (in megabytes) by the registry. // Validate it. // if (MmVirtualBias == 0) { // // Round the system view size (in megabytes) to a PDE multiple. // MmSystemViewSize = MI_ROUND_TO_SIZE (MmSystemViewSize, MM_MB_MAPPED_BY_PDE); // // NT64 locates system views just after systemwide paged pool, // so the size of the system views are not limited by session // space. Arbitrarily make the maximum a PPE's worth. // // // NT32 shares system view VA space with session VA space due // to the shortage of virtual addresses. Thus increasing the // system view size means potentially decreasing the maximum // session space size. // SystemViewMax = (MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE) / (1024*1024); #if !defined(_WIN64) // // Ensure at least enough space is left for // the standard default session layout. // SystemViewMax -= (MmSessionSize / (1024*1024)); #endif // // Note a view size of -1 will be rounded to zero. Treat -1 // as requesting the maximum. // if ((MmSystemViewSize > SystemViewMax) || (MmSystemViewSize == 0)) { MmSystemViewSize = SystemViewMax; } MmSystemViewSize *= (1024*1024); } else { MmSystemViewSize = DefaultSystemViewSize; } } #if defined(_WIN64) HydraSpaceUsedForSystemViews = 0; #else HydraSpaceUsedForSystemViews = MmSystemViewSize; #endif MiSessionImageEnd = SessionEnd; // // Select reasonable Hydra image, pool and view virtual sizes. // A PDE of virtual space is the minimum size allowed for each type. // if (MmVirtualBias == 0) { if (MmSessionImageSize < MM_MB_MAPPED_BY_PDE) { MmSessionImageSize = MI_SESSION_DEFAULT_IMAGE_SIZE; } else { // // The Hydra image size has been specified (in megabytes) // by the registry. // // Round it to a PDE multiple and validate it. // MmSessionImageSize = MI_ROUND_TO_SIZE (MmSessionImageSize, MM_MB_MAPPED_BY_PDE); HydraImageMax = (MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE - HydraSpaceUsedForSystemViews - (MmSessionSize - MI_SESSION_DEFAULT_IMAGE_SIZE)) / (1024*1024); // // Note a view size of -1 will be rounded to zero. // Treat -1 as requesting the maximum. // if ((MmSessionImageSize > HydraImageMax) || (MmSessionImageSize == 0)) { MmSessionImageSize = HydraImageMax; } MmSessionImageSize *= (1024*1024); MmSessionSize -= MI_SESSION_DEFAULT_IMAGE_SIZE; MmSessionSize += MmSessionImageSize; } MiSessionImageStart = SessionEnd - MmSessionImageSize; // // The session image start and size has been established. // // Now initialize the session pool and view ranges which lie // virtually below it. // if (MmSessionViewSize < MM_MB_MAPPED_BY_PDE) { MmSessionViewSize = MI_SESSION_DEFAULT_VIEW_SIZE; } else { // // The Hydra view size has been specified (in megabytes) // by the registry. Validate it. // // Round the Hydra view size to a PDE multiple. // MmSessionViewSize = MI_ROUND_TO_SIZE (MmSessionViewSize, MM_MB_MAPPED_BY_PDE); HydraViewMax = (MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE - HydraSpaceUsedForSystemViews - (MmSessionSize - MI_SESSION_DEFAULT_VIEW_SIZE)) / (1024*1024); // // Note a view size of -1 will be rounded to zero. // Treat -1 as requesting the maximum. // if ((MmSessionViewSize > HydraViewMax) || (MmSessionViewSize == 0)) { MmSessionViewSize = HydraViewMax; } MmSessionViewSize *= (1024*1024); MmSessionSize -= MI_SESSION_DEFAULT_VIEW_SIZE; MmSessionSize += MmSessionViewSize; } MiSessionViewStart = SessionEnd - MmSessionImageSize - MI_SESSION_SPACE_WS_SIZE - MI_SESSION_SPACE_STRUCT_SIZE - MmSessionViewSize; // // The session view start and size has been established. // // Now initialize the session pool start and size which lies // virtually just below it. // MiSessionPoolEnd = MiSessionViewStart; if (MmSessionPoolSize < MM_MB_MAPPED_BY_PDE) { #if !defined(_WIN64) // // Professional and below use systemwide paged pool for session // allocations (this decision is made in win32k.sys). Server // and above use real session pool and 16mb isn't enough to // play high end game applications, etc. Since we're not // booted /3GB, try for an additional 16mb now. // if ((MmSessionPoolSize == 0) && (MmProductType != 0x00690057)) { HydraPoolMax = MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE - HydraSpaceUsedForSystemViews - MmSessionSize; if (HydraPoolMax >= 2 * MI_SESSION_DEFAULT_POOL_SIZE) { MmSessionPoolSize = 2 * MI_SESSION_DEFAULT_POOL_SIZE; MmSessionSize -= MI_SESSION_DEFAULT_POOL_SIZE; MmSessionSize += MmSessionPoolSize; } else { MmSessionPoolSize = MI_SESSION_DEFAULT_POOL_SIZE; } } else #endif MmSessionPoolSize = MI_SESSION_DEFAULT_POOL_SIZE; } else { // // The Hydra pool size has been specified (in megabytes) // by the registry. Validate it. // // Round the Hydra pool size to a PDE multiple. // MmSessionPoolSize = MI_ROUND_TO_SIZE (MmSessionPoolSize, MM_MB_MAPPED_BY_PDE); HydraPoolMax = (MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE - HydraSpaceUsedForSystemViews - (MmSessionSize - MI_SESSION_DEFAULT_POOL_SIZE)) / (1024*1024); // // Note a view size of -1 will be rounded to zero. // Treat -1 as requesting the maximum. // if ((MmSessionPoolSize > HydraPoolMax) || (MmSessionPoolSize == 0)) { MmSessionPoolSize = HydraPoolMax; } MmSessionPoolSize *= (1024*1024); MmSessionSize -= MI_SESSION_DEFAULT_POOL_SIZE; MmSessionSize += MmSessionPoolSize; } MiSessionPoolStart = MiSessionPoolEnd - MmSessionPoolSize; MmSessionBase = (ULONG_PTR) MiSessionPoolStart; #if defined (_WIN64) // // Session special pool immediately follows session regular pool // assuming the user has enabled either the verifier or special // pool. // if ((MmVerifyDriverBufferLength != (ULONG)-1) || ((MmSpecialPoolTag != 0) && (MmSpecialPoolTag != (ULONG)-1))) { MmSessionSize = MI_SESSION_SPACE_MAXIMUM_TOTAL_SIZE; MmSessionSpecialPoolEnd = (PVOID) MiSessionPoolStart; MmSessionBase = MM_SESSION_SPACE_DEFAULT; MmSessionSpecialPoolStart = (PVOID) MmSessionBase; } #endif ASSERT (MmSessionBase + MmSessionSize == SessionEnd); MiSessionSpaceEnd = SessionEnd; MiSessionSpacePageTables = (ULONG)(MmSessionSize / MM_VA_MAPPED_BY_PDE); #if !defined (_WIN64) MiSystemViewStart = MmSessionBase - MmSystemViewSize; #endif } else { // // When booted /3GB, no size overrides are allowed due to the // already severely limited virtual address space. // Initialize the other Hydra variables after the system cache. // MmSessionViewSize = MI_SESSION_DEFAULT_VIEW_SIZE; MmSessionPoolSize = MI_SESSION_DEFAULT_POOL_SIZE; MmSessionImageSize = MI_SESSION_DEFAULT_IMAGE_SIZE; MiSessionImageStart = MiSessionImageEnd - MmSessionImageSize; } // // Set the highest section base address. // // N.B. In 32-bit systems this address must be 2gb or less even for // systems that run with 3gb enabled. Otherwise, it would not // be possible to map based sections identically in all processes. // MmHighSectionBase = ((PCHAR)MmHighestUserAddress - 0x800000); MaximumSystemCacheSize = (MM_SYSTEM_CACHE_END - MM_SYSTEM_CACHE_START) >> PAGE_SHIFT; #if defined(_X86_) // // If boot.ini specified a sane number of MB that the administrator // wants to use for user virtual address space then use it. // UserVaLimit = 0; ReducedUserVaOption = strstr(LoaderBlock->LoadOptions, "USERVA"); if (ReducedUserVaOption != NULL) { ReducedUserVaOption = strstr(ReducedUserVaOption,"="); if (ReducedUserVaOption != NULL) { UserVaLimit = atol(ReducedUserVaOption+1); UserVaLimit = MI_ROUND_TO_SIZE (UserVaLimit, ((MM_VA_MAPPED_BY_PDE) / (1024*1024))); } // // Ignore the USERVA switch if the limit is too small. // if (UserVaLimit <= (2048 + 16)) { UserVaLimit = 0; } } if (MmVirtualBias != 0) { // // If the size of the boot image (likely due to a large registry) // overflows into where paged pool would normally start, then // move paged pool up now. This costs virtual address space (ie: // performance) but more importantly, allows the system to boot. // if (MmBootImageSize > 16 * 1024 * 1024) { MmPagedPoolStart = (PVOID)((PCHAR)MmPagedPoolStart + (MmBootImageSize - 16 * 1024 * 1024)); ASSERT (((ULONG_PTR)MmPagedPoolStart % MM_VA_MAPPED_BY_PDE) == 0); } // // The system has been biased to an alternate base address to // allow 3gb of user address space, set the user probe address // and the maximum system cache size. // // If the system has been biased to an alternate base address to // allow 3gb of user address space, then set the user probe address // and the maximum system cache size. if ((UserVaLimit > 2048) && (UserVaLimit < 3072)) { // // Use any space between the maximum user virtual address // and the system for extra system PTEs. // // Convert input MB to bytes. // UserVaLimit -= 2048; UserVaLimit *= (1024*1024); // // Don't let the user specify a value which would cause us to // prematurely overwrite portions of the kernel & loader block. // if (UserVaLimit < MmBootImageSize) { UserVaLimit = MmBootImageSize; } } else { UserVaLimit = 0x40000000; } MmHighestUserAddress = ((PCHAR)MmHighestUserAddress + UserVaLimit); MmSystemRangeStart = ((PCHAR)MmSystemRangeStart + UserVaLimit); MmUserProbeAddress += UserVaLimit; MiMaximumWorkingSet += UserVaLimit >> PAGE_SHIFT; if (UserVaLimit != 0x40000000) { MiUseMaximumSystemSpace = (ULONG_PTR)MmSystemRangeStart; MiUseMaximumSystemSpaceEnd = 0xC0000000; } MiHighestUserPte = MiGetPteAddress (MmHighestUserAddress); MiHighestUserPde = MiGetPdeAddress (MmHighestUserAddress); // // Moving to 3GB means moving session space to just above // the system cache (and lowering the system cache max size // accordingly). Here's the visual: // // +------------------------------------+ // C1000000 | System cache resides here | // | and grows upward. | // | | | // | | | // | \/ | // | | // +------------------------------------+ // | Session space (Hydra). | // +------------------------------------+ // | Systemwide global mapped views. | // +------------------------------------+ // | | // | ^ | // | | | // | | | // | | // | Kernel, HAL & boot loaded images | // | grow downward from E1000000. | // | Total size is specified by | // | LoaderBlock->u.I386.BootImageSize. | // | Note only ntldrs after Build 2195 | // | are capable of loading the boot | // | images in descending order from | // | a hardcoded E1000000 on down. | // E1000000 +------------------------------------+ // MaximumSystemCacheSize -= MmBootImageSize >> PAGE_SHIFT; MaximumSystemCacheSize -= MmSessionSize >> PAGE_SHIFT; MaximumSystemCacheSize -= MmSystemViewSize >> PAGE_SHIFT; MmSessionBase = (ULONG_PTR)(MM_SYSTEM_CACHE_START + (MaximumSystemCacheSize << PAGE_SHIFT)); MiSystemViewStart = MmSessionBase + MmSessionSize; MiSessionPoolStart = MmSessionBase; MiSessionPoolEnd = MiSessionPoolStart + MmSessionPoolSize; MiSessionViewStart = MiSessionPoolEnd; MiSessionSpaceEnd = (ULONG_PTR)MmSessionBase + MmSessionSize; MiSessionSpacePageTables = MmSessionSize / MM_VA_MAPPED_BY_PDE; MiSessionImageEnd = MiSessionSpaceEnd; MiSessionImageStart = MiSessionImageEnd - MmSessionImageSize; } else if ((UserVaLimit >= 64) && (UserVaLimit < 2048)) { // // Convert input MB to bytes. // UserVaLimit *= (1024*1024); ReductionInBytes = 0x80000000 - UserVaLimit; MmHighestUserAddress = ((PCHAR)MmHighestUserAddress - ReductionInBytes); MmSystemRangeStart = ((PCHAR)MmSystemRangeStart - ReductionInBytes); MmUserProbeAddress -= ReductionInBytes; MiMaximumWorkingSet -= ReductionInBytes >> PAGE_SHIFT; MiUseMaximumSystemSpace = (ULONG_PTR)MmSystemRangeStart; MiUseMaximumSystemSpaceEnd = (ULONG_PTR)MiUseMaximumSystemSpace + ReductionInBytes; MmHighSectionBase = (PVOID)((PCHAR)MmHighSectionBase - ReductionInBytes); MiHighestUserPte = MiGetPteAddress (MmHighestUserAddress); MiHighestUserPde = MiGetPdeAddress (MmHighestUserAddress); } #else #if !defined (_WIN64) MaximumSystemCacheSize -= (MmSystemViewSize >> PAGE_SHIFT); #endif #endif // // Initialize some global session variables. // MmSessionSpace = (PMM_SESSION_SPACE)((ULONG_PTR)MmSessionBase + MmSessionSize - MmSessionImageSize - MI_SESSION_SPACE_STRUCT_SIZE); MiSessionImagePteStart = MiGetPteAddress ((PVOID) MiSessionImageStart); MiSessionImagePteEnd = MiGetPteAddress ((PVOID) MiSessionImageEnd); MiSessionBasePte = MiGetPteAddress ((PVOID)MmSessionBase); MiSessionSpaceWs = MiSessionViewStart + MmSessionViewSize; MiSessionLastPte = MiGetPteAddress ((PVOID)MiSessionSpaceEnd); #if DBG // // A few sanity checks to ensure things are as they should be. // if ((sizeof(CONTROL_AREA) % 8) != 0) { DbgPrint("control area list is not a quadword sized structure\n"); } if ((sizeof(SUBSECTION) % 8) != 0) { DbgPrint("subsection list is not a quadword sized structure\n"); } // // Some checks to make sure prototype PTEs can be placed in // either paged or nonpaged (prototype PTEs for paged pool are here) // can be put into PTE format. // PointerPte = (PMMPTE)MmPagedPoolStart; Pointer.u.Long = MiProtoAddressForPte (PointerPte); TempPte = Pointer; PointerPde = MiPteToProto(&TempPte); if (PointerPte != PointerPde) { DbgPrint("unable to map start of paged pool as prototype PTE %p %p\n", PointerPde, PointerPte); } PointerPte = (PMMPTE)((ULONG_PTR)MM_NONPAGED_POOL_END & ~((1 << PTE_SHIFT) - 1)); Pointer.u.Long = MiProtoAddressForPte (PointerPte); TempPte = Pointer; PointerPde = MiPteToProto(&TempPte); if (PointerPte != PointerPde) { DbgPrint("unable to map end of nonpaged pool as prototype PTE %p %p\n", PointerPde, PointerPte); } PointerPte = (PMMPTE)(((ULONG_PTR)NON_PAGED_SYSTEM_END - 0x37000 + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)); for (j = 0; j < 20; j += 1) { Pointer.u.Long = MiProtoAddressForPte (PointerPte); TempPte = Pointer; PointerPde = MiPteToProto(&TempPte); if (PointerPte != PointerPde) { DbgPrint("unable to map end of nonpaged pool as prototype PTE %p %p\n", PointerPde, PointerPte); } PointerPte += 1; } PointerPte = (PMMPTE)(((ULONG_PTR)MM_NONPAGED_POOL_END - 0x133448) & ~(ULONG_PTR)7); Pointer.u.Long = MiGetSubsectionAddressForPte (PointerPte); TempPte = Pointer; PointerPde = (PMMPTE)MiGetSubsectionAddress(&TempPte); if (PointerPte != PointerPde) { DbgPrint("unable to map end of nonpaged pool as section PTE %p %p\n", PointerPde, PointerPte); MiFormatPte(&TempPte); } // // End of sanity checks. // #endif if (MmEnforceWriteProtection) { MiPteStr[0] = (CHAR)1; } InitializeListHead (&MmLoadedUserImageList); InitializeListHead (&MmLockConflictList); InitializeListHead (&MmProtectedPteList); KeInitializeSpinLock (&MmProtectedPteLock); MmCriticalSectionTimeout.QuadPart = Int32x32To64( MmCritsectTimeoutSeconds, -10000000); // // Initialize System Address Space creation mutex. // ExInitializeFastMutex (&MmSectionCommitMutex); ExInitializeFastMutex (&MmSectionBasedMutex); ExInitializeFastMutex (&MmDynamicMemoryMutex); KeInitializeMutant (&MmSystemLoadLock, FALSE); KeInitializeEvent (&MmAvailablePagesEvent, NotificationEvent, TRUE); KeInitializeEvent (&MmAvailablePagesEventMedium, NotificationEvent, TRUE); KeInitializeEvent (&MmAvailablePagesEventHigh, NotificationEvent, TRUE); KeInitializeEvent (&MmMappedFileIoComplete, NotificationEvent, FALSE); KeInitializeEvent (&MmZeroingPageEvent, SynchronizationEvent, FALSE); KeInitializeEvent (&MmCollidedFlushEvent, NotificationEvent, FALSE); KeInitializeEvent (&MmCollidedLockEvent, NotificationEvent, FALSE); KeInitializeEvent (&MiMappedPagesTooOldEvent, NotificationEvent, FALSE); KeInitializeDpc (&MiModifiedPageWriterTimerDpc, MiModifiedPageWriterTimerDispatch, NULL); KeInitializeTimerEx (&MiModifiedPageWriterTimer, SynchronizationTimer); MiModifiedPageLife.QuadPart = Int32x32To64( MmModifiedPageLifeInSeconds, -10000000); InitializeListHead (&MmWorkingSetExpansionHead.ListHead); InitializeSListHead (&MmDeadStackSListHead); InitializeSListHead (&MmEventCountSListHead); InitializeSListHead (&MmInPageSupportSListHead); MmZeroingPageThreadActive = FALSE; MiMemoryLicense (LoaderBlock); // // include all memory types ... // for (i = 0; i < LoaderMaximum; i += 1) { IncludeType[i] = TRUE; } // // ... expect these.. // IncludeType[LoaderBad] = FALSE; IncludeType[LoaderFirmwarePermanent] = FALSE; IncludeType[LoaderSpecialMemory] = FALSE; IncludeType[LoaderBBTMemory] = FALSE; // // Compute number of pages in the system. // NumberOfPages = MiPagesInLoaderBlock (LoaderBlock, IncludeType); #if defined (_MI_MORE_THAN_4GB_) Mm64BitPhysicalAddress = TRUE; #endif // // When safebooting, don't enable special pool, the verifier or any // other options that track corruption regardless of registry settings. // if (strstr(LoaderBlock->LoadOptions, SAFEBOOT_LOAD_OPTION_A)) { MmVerifyDriverBufferLength = (ULONG)-1; MiVerifyAllDrivers = 0; MmVerifyDriverLevel = 0; MmDontVerifyRandomDrivers = TRUE; MmSpecialPoolTag = (ULONG)-1; MmSnapUnloads = FALSE; MmProtectFreedNonPagedPool = FALSE; MmEnforceWriteProtection = 0; MmTrackLockedPages = FALSE; MmTrackPtes = 0; #if defined (_WIN64) MmSessionSpecialPoolEnd = NULL; MmSessionSpecialPoolStart = NULL; #endif SharedUserData->SafeBootMode = TRUE; } else { MiTriageSystem (LoaderBlock); } SystemPteMultiplier = 0; if (MmNumberOfSystemPtes == 0) { #if defined (_WIN64) // // 64-bit NT is not constrained by virtual address space. No // tradeoffs between nonpaged pool, paged pool and system PTEs // need to be made. So just allocate PTEs on a linear scale as // a function of the amount of RAM. // // For example on a Hydra NT64, 4gb of RAM gets 128gb of PTEs. // The page table cost is the inversion of the multiplier based // on the PTE_PER_PAGE. // if (ExpMultiUserTS == TRUE) { SystemPteMultiplier = 32; } else { SystemPteMultiplier = 16; } if (NumberOfPages < 0x8000) { SystemPteMultiplier >>= 1; } #else if (NumberOfPages < MM_MEDIUM_SYSTEM) { MmNumberOfSystemPtes = MM_MINIMUM_SYSTEM_PTES; } else { MmNumberOfSystemPtes = MM_DEFAULT_SYSTEM_PTES; if (NumberOfPages > 8192) { MmNumberOfSystemPtes += MmNumberOfSystemPtes; // // Any reasonable Hydra machine gets the maximum. // if (ExpMultiUserTS == TRUE) { MmNumberOfSystemPtes = MM_MAXIMUM_SYSTEM_PTES; } } } #endif } else if (MmNumberOfSystemPtes == (ULONG)-1) { // // This registry setting indicates the maximum number of // system PTEs possible for this machine must be allocated. // Snap this for later reference. // MiRequestedSystemPtes = MmNumberOfSystemPtes; #if defined (_WIN64) SystemPteMultiplier = 256; #else MmNumberOfSystemPtes = MM_MAXIMUM_SYSTEM_PTES; #endif } if (SystemPteMultiplier != 0) { if (NumberOfPages * SystemPteMultiplier > MM_MAXIMUM_SYSTEM_PTES) { MmNumberOfSystemPtes = MM_MAXIMUM_SYSTEM_PTES; } else { MmNumberOfSystemPtes = (ULONG)(NumberOfPages * SystemPteMultiplier); } } if (MmNumberOfSystemPtes > MM_MAXIMUM_SYSTEM_PTES) { MmNumberOfSystemPtes = MM_MAXIMUM_SYSTEM_PTES; } if (MmNumberOfSystemPtes < MM_MINIMUM_SYSTEM_PTES) { MmNumberOfSystemPtes = MM_MINIMUM_SYSTEM_PTES; } if (MmHeapSegmentReserve == 0) { MmHeapSegmentReserve = 1024 * 1024; } if (MmHeapSegmentCommit == 0) { MmHeapSegmentCommit = PAGE_SIZE * 2; } if (MmHeapDeCommitTotalFreeThreshold == 0) { MmHeapDeCommitTotalFreeThreshold = 64 * 1024; } if (MmHeapDeCommitFreeBlockThreshold == 0) { MmHeapDeCommitFreeBlockThreshold = PAGE_SIZE; } #ifndef NO_POOL_CHECKS MiInitializeSpecialPoolCriteria (); #endif // // If the registry indicates drivers are in the suspect list, // extra system PTEs need to be allocated to support special pool // for their allocations. // if ((MmVerifyDriverBufferLength != (ULONG)-1) || ((MmSpecialPoolTag != 0) && (MmSpecialPoolTag != (ULONG)-1))) { MmNumberOfSystemPtes += MM_SPECIAL_POOL_PTES; } MmNumberOfSystemPtes += BBTPagesToReserve; #if defined(_X86_) // // The allocation preference key must be carefully managed. This is // because doing every allocation top-down can caused failures if // an ntdll process startup allocation (like the stack trace database) // gets a high address which then causes a subsequent system DLL rebase // collision. // // This is circumvented as follows: // // 1. For 32-bit machines, the allocation preference key is only // useful when booted /3GB as only then can this key help track // down apps with high virtual address bit sign extension problems. // In 3GB mode, the system DLLs are based just below 2GB so ntdll // would have to allocate more than 1GB of VA space before this // becomes a problem. So really the problem can only occur for // machines in 2GB mode and since the key doesn't help these // machines anyway, just turn it off in these cases. // // 2. For 64-bit machines, there is plenty of VA space above the // addresses system DLLs are based at so it is a non-issue. // EXCEPT for wow64 binaries which run in sandboxed 2GB address // spaces. Explicit checks are made to detect a wow64 process in // the Mm APIs which check this key and the key is ignored in // this case as it doesn't provide any sign extension help and // therefore we don't allow it to burn up any valuable VA space // which could cause a collision. // if (MmVirtualBias == 0) { MmAllocationPreference = 0; } #endif if (MmAllocationPreference != 0) { MmAllocationPreference = MEM_TOP_DOWN; } ExInitializeResourceLite (&MmSystemWsLock); MiInitializeDriverVerifierList (LoaderBlock); // // Set the initial commit page limit high enough so initial pool // allocations (which happen in the machine dependent init) can // succeed. // MmTotalCommitLimit = _2gb / PAGE_SIZE; MmTotalCommitLimitMaximum = MmTotalCommitLimit; // // Pick a reasonable size for the default prototype PTE allocation // chunk size. Make sure it's always a PAGE_SIZE multiple. The // registry entry is treated as the number of 1K chunks. // if (MmAllocationFragment == 0) { AutosizingFragment = TRUE; MmAllocationFragment = MM_ALLOCATION_FRAGMENT; #if !defined (_WIN64) if (NumberOfPages < 64 * 1024) { MmAllocationFragment = MM_ALLOCATION_FRAGMENT / 4; } else if (NumberOfPages < 256 * 1024) { MmAllocationFragment = MM_ALLOCATION_FRAGMENT / 2; } #endif } else { // // Convert the registry entry from 1K chunks into bytes. // Then round it to a PAGE_SIZE multiple. Finally bound it // reasonably. // AutosizingFragment = FALSE; MmAllocationFragment *= 1024; MmAllocationFragment = ROUND_TO_PAGES (MmAllocationFragment); if (MmAllocationFragment > MM_ALLOCATION_FRAGMENT) { MmAllocationFragment = MM_ALLOCATION_FRAGMENT; } else if (MmAllocationFragment < PAGE_SIZE) { MmAllocationFragment = PAGE_SIZE; } } MiInitializeIoTrackers (); MiInitializeCacheOverrides (); // // Initialize the machine dependent portion of the hardware. // MiInitMachineDependent (LoaderBlock); MmPhysicalMemoryBlock = MmInitializeMemoryLimits (LoaderBlock, IncludeType, NULL); if (MmPhysicalMemoryBlock == NULL) { KeBugCheckEx (INSTALL_MORE_MEMORY, MmNumberOfPhysicalPages, MmLowestPhysicalPage, MmHighestPhysicalPage, 0x100); } #if defined(_X86_) MiReportPhysicalMemory (); #endif #if defined (_MI_MORE_THAN_4GB_) if (MiNoLowMemory != 0) { MiRemoveLowPages (0); } #endif // // Initialize listhead, spinlock and semaphore for // segment dereferencing thread. // KeInitializeSpinLock (&MmDereferenceSegmentHeader.Lock); InitializeListHead (&MmDereferenceSegmentHeader.ListHead); KeInitializeSemaphore (&MmDereferenceSegmentHeader.Semaphore, 0, MAXLONG); InitializeListHead (&MmUnusedSegmentList); InitializeListHead (&MmUnusedSubsectionList); KeInitializeEvent (&MmUnusedSegmentCleanup, NotificationEvent, FALSE); MiInitializeCommitment (); MiInitializePfnTracing (); #if defined(_X86_) // // Virtual bias indicates the offset that needs to be added to // 0x80000000 to get to the start of the loaded images. Update it // now to indicate the offset to MmSessionBase as that is the lowest // system address that process creation needs to make sure to duplicate. // // This is not done until after machine dependent initialization runs // as that initialization relies on the original meaning of VirtualBias. // // Note if the system is booted with both /3GB & /USERVA, then system // PTEs will be allocated below virtual 3GB and that will end up being // the lowest system address the process creation needs to duplicate. // if (MmVirtualBias != 0) { MmVirtualBias = (ULONG_PTR)MmSessionBase - CODE_START; } #endif if (MmMirroring & MM_MIRRORING_ENABLED) { #if defined (_WIN64) // // All page frame numbers must fit in 32 bits because the bitmap // package is currently 32-bit. // // The bitmaps are deliberately not initialized as each mirroring // must reinitialize them anyway. // if (MmHighestPossiblePhysicalPage + 1 < _4gb) { #endif MiCreateBitMap (&MiMirrorBitMap, MmHighestPossiblePhysicalPage + 1, NonPagedPool); if (MiMirrorBitMap != NULL) { MiCreateBitMap (&MiMirrorBitMap2, MmHighestPossiblePhysicalPage + 1, NonPagedPool); if (MiMirrorBitMap2 == NULL) { MiRemoveBitMap (&MiMirrorBitMap); } } #if defined (_WIN64) } #endif } #if !defined (_WIN64) if ((AutosizingFragment == TRUE) && (NumberOfPages >= 256 * 1024)) { // // This is a system with at least 1GB of RAM. Presumably it // will be used to cache many files. Maybe we should factor in // pool size here and adjust it accordingly. // MmAllocationFragment; } #endif MiReloadBootLoadedDrivers (LoaderBlock); #if defined (_MI_MORE_THAN_4GB_) if (MiNoLowMemory != 0) { MiRemoveLowPages (1); } #endif MiInitializeVerifyingComponents (LoaderBlock); // // Setup the system size as small, medium, or large depending // on memory available. // // For internal MM tuning, the following applies // // 12Mb is small // 12-19 is medium // > 19 is large // // // For all other external tuning, // < 19 is small // 19 - 31 is medium for workstation // 19 - 63 is medium for server // >= 32 is large for workstation // >= 64 is large for server // if (MmNumberOfPhysicalPages <= MM_SMALL_SYSTEM) { MmSystemSize = MmSmallSystem; MmMaximumDeadKernelStacks = 0; MmModifiedPageMinimum = 40; MmModifiedPageMaximum = 100; MmDataClusterSize = 0; MmCodeClusterSize = 1; MmReadClusterSize = 2; MmInPageSupportMinimum = 2; } else if (MmNumberOfPhysicalPages <= MM_MEDIUM_SYSTEM) { MmSystemSize = MmSmallSystem; MmMaximumDeadKernelStacks = 2; MmModifiedPageMinimum = 80; MmModifiedPageMaximum = 150; MmSystemCacheWsMinimum += 100; MmSystemCacheWsMaximum += 150; MmDataClusterSize = 1; MmCodeClusterSize = 2; MmReadClusterSize = 4; MmInPageSupportMinimum = 3; } else { MmSystemSize = MmMediumSystem; MmMaximumDeadKernelStacks = 5; MmModifiedPageMinimum = 150; MmModifiedPageMaximum = 300; MmSystemCacheWsMinimum += 400; MmSystemCacheWsMaximum += 800; MmDataClusterSize = 3; MmCodeClusterSize = 7; MmReadClusterSize = 7; MmInPageSupportMinimum = 4; } if (MmNumberOfPhysicalPages < ((24*1024*1024)/PAGE_SIZE)) { MmSystemCacheWsMinimum = 32; } if (MmNumberOfPhysicalPages >= ((32*1024*1024)/PAGE_SIZE)) { // // If we are on a workstation, 32Mb and above are considered // large systems. // if (MmProductType == 0x00690057) { MmSystemSize = MmLargeSystem; } else { // // For servers, 64Mb and greater is a large system // if (MmNumberOfPhysicalPages >= ((64*1024*1024)/PAGE_SIZE)) { MmSystemSize = MmLargeSystem; } } } if (MmNumberOfPhysicalPages > ((33*1024*1024)/PAGE_SIZE)) { MmModifiedPageMinimum = 400; MmModifiedPageMaximum = 800; MmSystemCacheWsMinimum += 500; MmSystemCacheWsMaximum += 900; MmInPageSupportMinimum += 4; } if (MmNumberOfPhysicalPages > ((220*1024*1024)/PAGE_SIZE)) { // bump max cache size a bit more if ( (LONG)MmSystemCacheWsMinimum < (LONG)((24*1024*1024) >> PAGE_SHIFT) && (LONG)MmSystemCacheWsMaximum < (LONG)((24*1024*1024) >> PAGE_SHIFT)){ MmSystemCacheWsMaximum = ((24*1024*1024) >> PAGE_SHIFT); } ASSERT ((LONG)MmSystemCacheWsMaximum > (LONG)MmSystemCacheWsMinimum); } else if (MmNumberOfPhysicalPages > ((110*1024*1024)/PAGE_SIZE)) { // bump max cache size a bit if ( (LONG)MmSystemCacheWsMinimum < (LONG)((16*1024*1024) >> PAGE_SHIFT) && (LONG)MmSystemCacheWsMaximum < (LONG)((16*1024*1024) >> PAGE_SHIFT)){ MmSystemCacheWsMaximum = ((16*1024*1024) >> PAGE_SHIFT); } ASSERT ((LONG)MmSystemCacheWsMaximum > (LONG)MmSystemCacheWsMinimum); } if (NT_SUCCESS (MmIsVerifierEnabled (&VerifierFlags))) { // // The verifier is enabled so don't defer any MDL unlocks because // without state, debugging driver bugs in this area is very // difficult. // DeferredMdlEntries = 0; } else if (MmNumberOfPhysicalPages > ((255*1024*1024)/PAGE_SIZE)) { DeferredMdlEntries = 32; } else if (MmNumberOfPhysicalPages > ((127*1024*1024)/PAGE_SIZE)) { DeferredMdlEntries = 8; } else { DeferredMdlEntries = 4; } #if defined(MI_MULTINODE) for (i = 0; i < KeNumberNodes; i += 1) { InitializeSListHead (&KeNodeBlock[i]->PfnDereferenceSListHead); KeNodeBlock[i]->PfnDeferredList = NULL; for (j = 0; j < DeferredMdlEntries; j += 1) { SingleListEntry = ExAllocatePoolWithTag (NonPagedPool, sizeof(MI_PFN_DEREFERENCE_CHUNK), 'mDmM'); if (SingleListEntry != NULL) { InterlockedPushEntrySList (&KeNodeBlock[i]->PfnDereferenceSListHead, SingleListEntry); } } } #else InitializeSListHead (&MmPfnDereferenceSListHead); for (j = 0; j < DeferredMdlEntries; j += 1) { SingleListEntry = ExAllocatePoolWithTag (NonPagedPool, sizeof(MI_PFN_DEREFERENCE_CHUNK), 'mDmM'); if (SingleListEntry != NULL) { InterlockedPushEntrySList (&MmPfnDereferenceSListHead, SingleListEntry); } } #endif ASSERT (SharedUserData->NumberOfPhysicalPages == 0); SharedUserData->NumberOfPhysicalPages = (ULONG) MmNumberOfPhysicalPages; // // Determine if we are on an AS system (Winnt is not AS). // if (MmProductType == 0x00690057) { SharedUserData->NtProductType = NtProductWinNt; MmProductType = 0; MmThrottleTop = 250; MmThrottleBottom = 30; } else { if (MmProductType == 0x0061004c) { SharedUserData->NtProductType = NtProductLanManNt; } else { SharedUserData->NtProductType = NtProductServer; } MmProductType = 1; MmThrottleTop = 450; MmThrottleBottom = 80; MmMinimumFreePages = 81; MmInPageSupportMinimum += 8; } MiAdjustWorkingSetManagerParameters ((LOGICAL)(MmProductType == 0 ? TRUE : FALSE)); // // Set the ResidentAvailablePages to the number of available // pages minus the fluid value. // MmResidentAvailablePages = MmAvailablePages - MM_FLUID_PHYSICAL_PAGES; // // Subtract off the size of future nonpaged pool expansion // so that nonpaged pool will always be able to expand regardless of // prior system load activity. // MmResidentAvailablePages -= MiExpansionPoolPagesInitialCharge; // // Subtract off the size of the system cache working set. // MmResidentAvailablePages -= MmSystemCacheWsMinimum; MmResidentAvailableAtInit = MmResidentAvailablePages; if (MmResidentAvailablePages < 0) { #if DBG DbgPrint("system cache working set too big\n"); #endif return FALSE; } // // Initialize spin lock for allowing working set expansion. // KeInitializeSpinLock (&MmExpansionLock); ExInitializeFastMutex (&MmPageFileCreationLock); // // Initialize resource for extending sections. // ExInitializeResourceLite (&MmSectionExtendResource); ExInitializeResourceLite (&MmSectionExtendSetResource); // // Build the system cache structures. // StartPde = MiGetPdeAddress (MmSystemCacheWorkingSetList); PointerPte = MiGetPteAddress (MmSystemCacheWorkingSetList); #if (_MI_PAGING_LEVELS >= 3) TempPte = ValidKernelPte; #if (_MI_PAGING_LEVELS >= 4) StartPxe = MiGetPdeAddress(StartPde); if (StartPxe->u.Hard.Valid == 0) { // // Map in a page directory parent page for the system cache working // set. Note that we only populate one page table for this. // DirectoryFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (StartPxe)); TempPte.u.Hard.PageFrameNumber = DirectoryFrameIndex; *StartPxe = TempPte; MiInitializePfn (DirectoryFrameIndex, StartPxe, 1); MiFillMemoryPte (MiGetVirtualAddressMappedByPte(StartPxe), PAGE_SIZE, ZeroKernelPte.u.Long); } #endif StartPpe = MiGetPteAddress(StartPde); if (StartPpe->u.Hard.Valid == 0) { // // Map in a page directory page for the system cache working set. // Note that we only populate one page table for this. // DirectoryFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (StartPpe)); TempPte.u.Hard.PageFrameNumber = DirectoryFrameIndex; *StartPpe = TempPte; MiInitializePfn (DirectoryFrameIndex, StartPpe, 1); MiFillMemoryPte (MiGetVirtualAddressMappedByPte(StartPpe), PAGE_SIZE, ZeroKernelPte.u.Long); } #if (_MI_PAGING_LEVELS >= 4) // // The shared user data is already initialized and it shares the // page table page with the system cache working set list. // ASSERT (StartPde->u.Hard.Valid == 1); #else // // Map in a page table page. // ASSERT (StartPde->u.Hard.Valid == 0); PageFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (StartPde)); TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (StartPde, TempPte); MiInitializePfn (PageFrameIndex, StartPde, 1); MiFillMemoryPte (MiGetVirtualAddressMappedByPte (StartPde), PAGE_SIZE, ZeroKernelPte.u.Long); #endif StartPpe = MiGetPpeAddress(MmSystemCacheStart); StartPde = MiGetPdeAddress(MmSystemCacheStart); PointerPte = MiGetVirtualAddressMappedByPte (StartPde); #else #if !defined(_X86PAE_) ASSERT ((StartPde + 1) == MiGetPdeAddress (MmSystemCacheStart)); #endif #endif MaximumSystemCacheSizeTotal = MaximumSystemCacheSize; #if defined(_X86_) MaximumSystemCacheSizeTotal += MiMaximumSystemCacheSizeExtra; #endif // // Size the system cache based on the amount of physical memory. // i = (MmNumberOfPhysicalPages + 65) / 1024; if (i >= 4) { // // System has at least 4032 pages. Make the system // cache 128mb + 64mb for each additional 1024 pages. // MmSizeOfSystemCacheInPages = (PFN_COUNT)( ((128*1024*1024) >> PAGE_SHIFT) + ((i - 4) * ((64*1024*1024) >> PAGE_SHIFT))); if (MmSizeOfSystemCacheInPages > MaximumSystemCacheSizeTotal) { MmSizeOfSystemCacheInPages = MaximumSystemCacheSizeTotal; } } MmSystemCacheEnd = (PVOID)(((PCHAR)MmSystemCacheStart + MmSizeOfSystemCacheInPages * PAGE_SIZE) - 1); #if defined(_X86_) if (MmSizeOfSystemCacheInPages > MaximumSystemCacheSize) { ASSERT (MiMaximumSystemCacheSizeExtra != 0); MmSystemCacheEnd = (PVOID)(((PCHAR)MmSystemCacheStart + MaximumSystemCacheSize * PAGE_SIZE) - 1); MiSystemCacheStartExtra = (PVOID)MiExtraResourceStart; MiSystemCacheEndExtra = (PVOID)(((PCHAR)MiSystemCacheStartExtra + (MmSizeOfSystemCacheInPages - MaximumSystemCacheSize) * PAGE_SIZE) - 1); } else { MiSystemCacheStartExtra = MmSystemCacheStart; MiSystemCacheEndExtra = MmSystemCacheEnd; } #endif EndPde = MiGetPdeAddress(MmSystemCacheEnd); TempPte = ValidKernelPte; #if (_MI_PAGING_LEVELS >= 4) StartPxe = MiGetPxeAddress(MmSystemCacheStart); if (StartPxe->u.Hard.Valid == 0) { FirstPxe = TRUE; FirstPpe = TRUE; } else { FirstPxe = FALSE; FirstPpe = (StartPpe->u.Hard.Valid == 0) ? TRUE : FALSE; } #elif (_MI_PAGING_LEVELS >= 3) FirstPpe = (StartPpe->u.Hard.Valid == 0) ? TRUE : FALSE; #else DirectoryFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (MiGetPteAddress(PDE_BASE)); #endif LOCK_PFN (OldIrql); while (StartPde <= EndPde) { #if (_MI_PAGING_LEVELS >= 4) if (FirstPxe == TRUE || MiIsPteOnPpeBoundary(StartPde)) { FirstPxe = FALSE; StartPxe = MiGetPdeAddress(StartPde); // // Map in a page directory page. // DirectoryFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (StartPxe)); TempPte.u.Hard.PageFrameNumber = DirectoryFrameIndex; MI_WRITE_VALID_PTE (StartPxe, TempPte); MiInitializePfn (DirectoryFrameIndex, StartPxe, 1); MiFillMemoryPte (MiGetVirtualAddressMappedByPte(StartPxe), PAGE_SIZE, ZeroKernelPte.u.Long); } #endif #if (_MI_PAGING_LEVELS >= 3) if (FirstPpe == TRUE || MiIsPteOnPdeBoundary(StartPde)) { FirstPpe = FALSE; StartPpe = MiGetPteAddress(StartPde); // // Map in a page directory page. // DirectoryFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (StartPpe)); TempPte.u.Hard.PageFrameNumber = DirectoryFrameIndex; MI_WRITE_VALID_PTE (StartPpe, TempPte); MiInitializePfn (DirectoryFrameIndex, StartPpe, 1); MiFillMemoryPte (MiGetVirtualAddressMappedByPte(StartPpe), PAGE_SIZE, ZeroKernelPte.u.Long); } #endif ASSERT (StartPde->u.Hard.Valid == 0); // // Map in a page table page. // PageFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (StartPde)); TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (StartPde, TempPte); MiInitializePfn (PageFrameIndex, StartPde, 1); MiFillMemoryPte (MiGetVirtualAddressMappedByPte(StartPde), PAGE_SIZE, ZeroKernelPte.u.Long); StartPde += 1; } UNLOCK_PFN (OldIrql); // // Initialize the system cache. Only set the large system cache if // we have a large amount of physical memory. // if (MmLargeSystemCache != 0 && MmNumberOfPhysicalPages > 0x7FF0) { if ((MmAvailablePages > MmSystemCacheWsMaximum + ((64*1024*1024) >> PAGE_SHIFT))) { MmSystemCacheWsMaximum = MmAvailablePages - ((32*1024*1024) >> PAGE_SHIFT); ASSERT ((LONG)MmSystemCacheWsMaximum > (LONG)MmSystemCacheWsMinimum); } } if (MmSystemCacheWsMaximum > (MM_MAXIMUM_WORKING_SET - 5)) { MmSystemCacheWsMaximum = MM_MAXIMUM_WORKING_SET - 5; } if (MmSystemCacheWsMaximum > MmSizeOfSystemCacheInPages) { MmSystemCacheWsMaximum = MmSizeOfSystemCacheInPages; if ((MmSystemCacheWsMinimum + 500) > MmSystemCacheWsMaximum) { MmSystemCacheWsMinimum = MmSystemCacheWsMaximum - 500; } } MiInitializeSystemCache ((ULONG)MmSystemCacheWsMinimum, (ULONG)MmSystemCacheWsMaximum); MmAttemptForCantExtend.Segment = NULL; MmAttemptForCantExtend.RequestedExpansionSize = 1; MmAttemptForCantExtend.ActualExpansion = 0; MmAttemptForCantExtend.InProgress = FALSE; MmAttemptForCantExtend.PageFileNumber = MI_EXTEND_ANY_PAGEFILE; KeInitializeEvent (&MmAttemptForCantExtend.Event, NotificationEvent, FALSE); // // Now that we have booted far enough, replace the temporary // commit limits with real ones: set the initial commit page // limit to the number of available pages. This value is // updated as paging files are created. // MmTotalCommitLimit = MmAvailablePages; if (MmTotalCommitLimit > 1024) { MmTotalCommitLimit -= 1024; } MmTotalCommitLimitMaximum = MmTotalCommitLimit; // // Set maximum working set size to 512 pages less than the // total available memory. // MmMaximumWorkingSetSize = (WSLE_NUMBER)(MmAvailablePages - 512); if (MmMaximumWorkingSetSize > (MM_MAXIMUM_WORKING_SET - 5)) { MmMaximumWorkingSetSize = MM_MAXIMUM_WORKING_SET - 5; } // // Create the modified page writer event. // KeInitializeEvent (&MmModifiedPageWriterEvent, NotificationEvent, FALSE); // // Build paged pool. // MiBuildPagedPool (); // // Initialize the loaded module list. This cannot be done until // paged pool has been built. // if (MiInitializeLoadedModuleList (LoaderBlock) == FALSE) { #if DBG DbgPrint("Loaded module list initialization failed\n"); #endif return FALSE; } // // Initialize the unused segment threshold. Attempt to keep pool usage // below this percentage (by trimming the cache) if pool requests // can fail. // if (MmConsumedPoolPercentage == 0) { MmConsumedPoolPercentage = 80; } else if (MmConsumedPoolPercentage < 5) { MmConsumedPoolPercentage = 5; } else if (MmConsumedPoolPercentage > 100) { MmConsumedPoolPercentage = 100; } // // Add more system PTEs if this is a large memory system. // Note that 64 bit systems can determine the right value at the // beginning since there is no virtual address space crunch. // #if !defined (_WIN64) if (MmNumberOfPhysicalPages > ((127*1024*1024) >> PAGE_SHIFT)) { PMMPTE StartingPte; PointerPde = MiGetPdeAddress ((PCHAR)MmPagedPoolEnd + 1); StartingPte = MiGetPteAddress ((PCHAR)MmPagedPoolEnd + 1); j = 0; TempPte = ValidKernelPde; LOCK_PFN (OldIrql); while (PointerPde->u.Hard.Valid == 0) { MiChargeCommitmentCantExpand (1, TRUE); MM_TRACK_COMMIT (MM_DBG_COMMIT_EXTRA_SYSTEM_PTES, 1); PageFrameIndex = MiRemoveZeroPage ( MI_GET_PAGE_COLOR_FROM_PTE (PointerPde)); TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (PointerPde, TempPte); MiInitializePfn (PageFrameIndex, PointerPde, 1); PointerPde += 1; StartingPte += PAGE_SIZE / sizeof(MMPTE); j += PAGE_SIZE / sizeof(MMPTE); } UNLOCK_PFN (OldIrql); if (j != 0) { StartingPte = MiGetPteAddress ((PCHAR)MmPagedPoolEnd + 1); MmNonPagedSystemStart = MiGetVirtualAddressMappedByPte (StartingPte); MmNumberOfSystemPtes += j; MiAddSystemPtes (StartingPte, j, SystemPteSpace); MiIncrementSystemPtes (j); } } #endif #if defined (_MI_DEBUG_SUB) if (MiTrackSubs != 0) { MiSubsectionTraces = ExAllocatePoolWithTag (NonPagedPool, MiTrackSubs * sizeof (MI_SUB_TRACES), 'tCmM'); } #endif #if defined (_MI_DEBUG_DIRTY) if (MiTrackDirtys != 0) { MiDirtyTraces = ExAllocatePoolWithTag (NonPagedPool, MiTrackDirtys * sizeof (MI_DIRTY_TRACES), 'tCmM'); } #endif #if defined (_MI_DEBUG_DATA) if (MiTrackData != 0) { MiDataTraces = ExAllocatePoolWithTag (NonPagedPool, MiTrackData * sizeof (MI_DATA_TRACES), 'tCmM'); } #endif #if DBG if (MmDebug & MM_DBG_DUMP_BOOT_PTES) { MiDumpValidAddresses (); MiDumpPfn (); } #endif MmPageFaultNotifyRoutine = NULL; #ifdef _MI_MESSAGE_SERVER MiInitializeMessageQueue (); #endif return TRUE; } if (Phase == 1) { #if DBG MmDebug |= MM_DBG_CHECK_PFN_LOCK; #endif #if defined(_X86_) || defined(_AMD64_) MiInitMachineDependent (LoaderBlock); #endif MiMapBBTMemory(LoaderBlock); if (!MiSectionInitialization ()) { return FALSE; } Process = PsGetCurrentProcess (); if (Process->PhysicalVadList.Flink == NULL) { InitializeListHead (&Process->PhysicalVadList); } #if defined(MM_SHARED_USER_DATA_VA) // // Create double mapped page between kernel and user mode. // The PTE is deliberately allocated from paged pool so that // it will always have a PTE itself instead of being superpaged. // This way, checks throughout the fault handler can assume that // the PTE can be checked without having to special case this. // MmSharedUserDataPte = ExAllocatePoolWithTag (PagedPool, sizeof(MMPTE), ' mM'); if (MmSharedUserDataPte == NULL) { return FALSE; } PointerPte = MiGetPteAddress ((PVOID)KI_USER_SHARED_DATA); ASSERT (PointerPte->u.Hard.Valid == 1); PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte); MI_MAKE_VALID_PTE (TempPte, PageFrameIndex, MM_READONLY, PointerPte); *MmSharedUserDataPte = TempPte; Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); LOCK_PFN (OldIrql); Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE; UNLOCK_PFN (OldIrql); #ifdef _X86_ if (MmHighestUserAddress < (PVOID) MM_SHARED_USER_DATA_VA) { // // Install the PTE mapping now as faults will not because the // shared user data is in the system portion of the address space. // Note the pagetable page has already been allocated and locked // down. // // // Make the mapping user accessible. // ASSERT (MmSharedUserDataPte->u.Hard.Owner == 0); MmSharedUserDataPte->u.Hard.Owner = 1; PointerPde = MiGetPdeAddress (MM_SHARED_USER_DATA_VA); ASSERT (PointerPde->u.Hard.Owner == 0); PointerPde->u.Hard.Owner = 1; ASSERT (MiUseMaximumSystemSpace != 0); PointerPte = MiGetPteAddress (MM_SHARED_USER_DATA_VA); ASSERT (PointerPte->u.Hard.Valid == 0); MI_WRITE_VALID_PTE (PointerPte, *MmSharedUserDataPte); } #endif #endif MiSessionWideInitializeAddresses (); MiInitializeSessionWsSupport (); MiInitializeSessionIds (); // // Start the modified page writer. // InitializeObjectAttributes (&ObjectAttributes, NULL, 0, NULL, NULL); if (!NT_SUCCESS(PsCreateSystemThread( &ThreadHandle, THREAD_ALL_ACCESS, &ObjectAttributes, 0L, NULL, MiModifiedPageWriter, NULL ))) { return FALSE; } ZwClose (ThreadHandle); // // Initialize the low and high memory events. This must be done // before starting the working set manager. // if (MiInitializeMemoryEvents () == FALSE) { return FALSE; } // // Start the balance set manager. // // The balance set manager performs stack swapping and working // set management and requires two threads. // KeInitializeEvent (&MmWorkingSetManagerEvent, SynchronizationEvent, FALSE); InitializeObjectAttributes (&ObjectAttributes, NULL, 0, NULL, NULL); if (!NT_SUCCESS(PsCreateSystemThread( &ThreadHandle, THREAD_ALL_ACCESS, &ObjectAttributes, 0L, NULL, KeBalanceSetManager, NULL ))) { return FALSE; } ZwClose (ThreadHandle); if (!NT_SUCCESS(PsCreateSystemThread( &ThreadHandle, THREAD_ALL_ACCESS, &ObjectAttributes, 0L, NULL, KeSwapProcessOrStack, NULL ))) { return FALSE; } ZwClose (ThreadHandle); #ifndef NO_POOL_CHECKS MiInitializeSpecialPoolCriteria (); #endif #if defined(_X86_) MiEnableKernelVerifier (); #endif ExAcquireResourceExclusiveLite (&PsLoadedModuleResource, TRUE); NextEntry = PsLoadedModuleList.Flink; for ( ; NextEntry != &PsLoadedModuleList; NextEntry = NextEntry->Flink) { DataTableEntry = CONTAINING_RECORD(NextEntry, KLDR_DATA_TABLE_ENTRY, InLoadOrderLinks); NtHeaders = RtlImageNtHeader(DataTableEntry->DllBase); if ((NtHeaders->OptionalHeader.MajorOperatingSystemVersion >= 5) && (NtHeaders->OptionalHeader.MajorImageVersion >= 5)) { DataTableEntry->Flags |= LDRP_ENTRY_NATIVE; } MiWriteProtectSystemImage (DataTableEntry->DllBase); } ExReleaseResourceLite (&PsLoadedModuleResource); InterlockedDecrement (&MiTrimInProgressCount); return TRUE; } if (Phase == 2) { MiEnablePagingTheExecutive(); return TRUE; } return FALSE; } VOID MiMapBBTMemory ( IN PLOADER_PARAMETER_BLOCK LoaderBlock ) /*++ Routine Description: This function walks through the loader block's memory descriptor list and maps memory reserved for the BBT buffer into the system. The mapped PTEs are PDE-aligned and made user accessible. Arguments: LoaderBlock - Supplies a pointer to the system loader block. Return Value: None. Environment: Kernel Mode Only. System initialization. --*/ { PVOID Va; PMEMORY_ALLOCATION_DESCRIPTOR MemoryDescriptor; PLIST_ENTRY NextMd; PFN_NUMBER NumberOfPagesMapped; PFN_NUMBER NumberOfPages; PFN_NUMBER PageFrameIndex; PMMPTE PointerPte; PMMPTE PointerPde; PMMPTE LastPde; MMPTE TempPte; if (BBTPagesToReserve <= 0) { return; } // // Request enough PTEs such that protection can be applied to the PDEs. // NumberOfPages = (BBTPagesToReserve + (PTE_PER_PAGE - 1)) & ~(PTE_PER_PAGE - 1); PointerPte = MiReserveAlignedSystemPtes ((ULONG)NumberOfPages, SystemPteSpace, MM_VA_MAPPED_BY_PDE); if (PointerPte == NULL) { BBTPagesToReserve = 0; return; } // // Allow user access to the buffer. // PointerPde = MiGetPteAddress (PointerPte); LastPde = MiGetPteAddress (PointerPte + NumberOfPages); ASSERT (LastPde != PointerPde); do { TempPte = *PointerPde; TempPte.u.Long |= MM_PTE_OWNER_MASK; MI_WRITE_VALID_PTE (PointerPde, TempPte); PointerPde += 1; } while (PointerPde < LastPde); KeFlushEntireTb (TRUE, TRUE); Va = MiGetVirtualAddressMappedByPte (PointerPte); TempPte = ValidUserPte; NumberOfPagesMapped = 0; NextMd = LoaderBlock->MemoryDescriptorListHead.Flink; while (NextMd != &LoaderBlock->MemoryDescriptorListHead) { MemoryDescriptor = CONTAINING_RECORD(NextMd, MEMORY_ALLOCATION_DESCRIPTOR, ListEntry); if (MemoryDescriptor->MemoryType == LoaderBBTMemory) { PageFrameIndex = MemoryDescriptor->BasePage; NumberOfPages = MemoryDescriptor->PageCount; if (NumberOfPagesMapped + NumberOfPages > BBTPagesToReserve) { NumberOfPages = BBTPagesToReserve - NumberOfPagesMapped; } NumberOfPagesMapped += NumberOfPages; do { TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (PointerPte, TempPte); PointerPte += 1; PageFrameIndex += 1; NumberOfPages -= 1; } while (NumberOfPages); if (NumberOfPagesMapped == BBTPagesToReserve) { break; } } NextMd = MemoryDescriptor->ListEntry.Flink; } RtlZeroMemory(Va, BBTPagesToReserve << PAGE_SHIFT); // // Tell BBT_Init how many pages were allocated. // if (NumberOfPagesMapped < BBTPagesToReserve) { BBTPagesToReserve = (ULONG)NumberOfPagesMapped; } *(PULONG)Va = BBTPagesToReserve; // // At this point instrumentation code will detect the existence of // buffer and initialize the structures. // BBTBuffer = Va; PERFINFO_MMINIT_START(); } PPHYSICAL_MEMORY_DESCRIPTOR MmInitializeMemoryLimits ( IN PLOADER_PARAMETER_BLOCK LoaderBlock, IN PBOOLEAN IncludeType, IN OUT PPHYSICAL_MEMORY_DESCRIPTOR InputMemory OPTIONAL ) /*++ Routine Description: This function walks through the loader block's memory descriptor list and builds a list of contiguous physical memory blocks of the desired types. Arguments: LoaderBlock - Supplies a pointer the system loader block. IncludeType - Array of BOOLEANS of size LoaderMaximum. TRUE means include this type of memory in return. Memory - If non-NULL, supplies the physical memory blocks to place the search results in. If NULL, pool is allocated to hold the returned search results in - the caller must free this pool. Return Value: A pointer to the physical memory blocks for the requested search or NULL on failure. Environment: Kernel Mode Only. System initialization. --*/ { PLIST_ENTRY NextMd; ULONG i; ULONG InitialAllocation; PFN_NUMBER NextPage; PFN_NUMBER TotalPages; PPHYSICAL_MEMORY_DESCRIPTOR Memory; PPHYSICAL_MEMORY_DESCRIPTOR Memory2; PMEMORY_ALLOCATION_DESCRIPTOR MemoryDescriptor; InitialAllocation = 0; if (ARGUMENT_PRESENT (InputMemory)) { Memory = InputMemory; } else { // // The caller wants us to allocate the return result buffer. Size it // by allocating the maximum possibly needed as this should not be // very big (relatively). It is the caller's responsibility to free // this. Obviously this option can only be requested after pool has // been initialized. // NextMd = LoaderBlock->MemoryDescriptorListHead.Flink; while (NextMd != &LoaderBlock->MemoryDescriptorListHead) { InitialAllocation += 1; MemoryDescriptor = CONTAINING_RECORD(NextMd, MEMORY_ALLOCATION_DESCRIPTOR, ListEntry); NextMd = MemoryDescriptor->ListEntry.Flink; } Memory = ExAllocatePoolWithTag (NonPagedPool, sizeof(PHYSICAL_MEMORY_DESCRIPTOR) + sizeof(PHYSICAL_MEMORY_RUN) * (InitialAllocation - 1), 'lMmM'); if (Memory == NULL) { return NULL; } Memory->NumberOfRuns = InitialAllocation; } // // Walk through the memory descriptors and build the physical memory list. // i = 0; TotalPages = 0; NextPage = (PFN_NUMBER) -1; NextMd = LoaderBlock->MemoryDescriptorListHead.Flink; while (NextMd != &LoaderBlock->MemoryDescriptorListHead) { MemoryDescriptor = CONTAINING_RECORD(NextMd, MEMORY_ALLOCATION_DESCRIPTOR, ListEntry); if (MemoryDescriptor->MemoryType < LoaderMaximum && IncludeType [MemoryDescriptor->MemoryType]) { TotalPages += MemoryDescriptor->PageCount; // // Merge runs whenever possible. // if (MemoryDescriptor->BasePage == NextPage) { ASSERT (MemoryDescriptor->PageCount != 0); Memory->Run[i - 1].PageCount += MemoryDescriptor->PageCount; NextPage += MemoryDescriptor->PageCount; } else { Memory->Run[i].BasePage = MemoryDescriptor->BasePage; Memory->Run[i].PageCount = MemoryDescriptor->PageCount; NextPage = Memory->Run[i].BasePage + Memory->Run[i].PageCount; i += 1; } } NextMd = MemoryDescriptor->ListEntry.Flink; } ASSERT (i <= Memory->NumberOfRuns); if (i == 0) { // // Don't bother shrinking this as the caller will be freeing it // shortly as it is just an empty list. // Memory->Run[i].BasePage = 0; Memory->Run[i].PageCount = 0; } else if (!ARGUMENT_PRESENT (InputMemory)) { // // Shrink the buffer (if possible) now that the final size is known. // if (InitialAllocation > i) { Memory2 = ExAllocatePoolWithTag (NonPagedPool, sizeof(PHYSICAL_MEMORY_DESCRIPTOR) + sizeof(PHYSICAL_MEMORY_RUN) * (i - 1), 'lMmM'); if (Memory2 != NULL) { RtlCopyMemory (Memory2->Run, Memory->Run, sizeof(PHYSICAL_MEMORY_RUN) * i); ExFreePool (Memory); Memory = Memory2; } } } Memory->NumberOfRuns = i; Memory->NumberOfPages = TotalPages; return Memory; } PFN_NUMBER MiPagesInLoaderBlock ( IN PLOADER_PARAMETER_BLOCK LoaderBlock, IN PBOOLEAN IncludeType ) /*++ Routine Description: This function walks through the loader block's memory descriptor list and returns the number of pages of the desired type. Arguments: LoaderBlock - Supplies a pointer the system loader block. IncludeType - Array of BOOLEANS of size LoaderMaximum. TRUE means include this type of memory in the returned count. Return Value: The number of pages of the requested type in the loader block list. Environment: Kernel Mode Only. System initialization. --*/ { PMEMORY_ALLOCATION_DESCRIPTOR MemoryDescriptor; PLIST_ENTRY NextMd; PFN_NUMBER TotalPages; // // Walk through the memory descriptors counting pages. // TotalPages = 0; NextMd = LoaderBlock->MemoryDescriptorListHead.Flink; while (NextMd != &LoaderBlock->MemoryDescriptorListHead) { MemoryDescriptor = CONTAINING_RECORD(NextMd, MEMORY_ALLOCATION_DESCRIPTOR, ListEntry); if (MemoryDescriptor->MemoryType < LoaderMaximum && IncludeType [MemoryDescriptor->MemoryType]) { TotalPages += MemoryDescriptor->PageCount; } NextMd = MemoryDescriptor->ListEntry.Flink; } return TotalPages; } static VOID MiMemoryLicense ( IN PLOADER_PARAMETER_BLOCK LoaderBlock ) /*++ Routine Description: This function walks through the loader block's memory descriptor list and based on the system's license, ensures only the proper amount of physical memory is used. Arguments: LoaderBlock - Supplies a pointer to the system loader block. Return Value: None. Environment: Kernel Mode Only. System initialization. --*/ { PLIST_ENTRY NextMd; PFN_NUMBER TotalPagesAllowed; PFN_NUMBER PageCount; ULONG VirtualBias; PMEMORY_ALLOCATION_DESCRIPTOR MemoryDescriptor; // // The default configuration gets a maximum of 4gb physical memory. // On PAE machines the system continues to operate in 8-byte PTE mode. // TotalPagesAllowed = MI_DEFAULT_MAX_PAGES; // // If properly licensed (ie: DataCenter) and booted without the // 3gb switch, then use all available physical memory. // if (ExVerifySuite(DataCenter) == TRUE) { // // Note MmVirtualBias has not yet been initialized at the time of the // first call to this routine, so use the LoaderBlock directly. // #if defined(_X86_) VirtualBias = LoaderBlock->u.I386.VirtualBias; #else VirtualBias = 0; #endif if (VirtualBias == 0) { // // Limit the maximum physical memory to the amount we have // actually physically seen in a machine inhouse. // TotalPagesAllowed = MI_DTC_MAX_PAGES; } else { // // The system is booting /3gb, so don't use more than 16gb of // physical memory. This ensures enough virtual space to map // the PFN database. // TotalPagesAllowed = MI_DTC_BOOTED_3GB_MAX_PAGES; } } else if ((MmProductType != 0x00690057) && (ExVerifySuite(Enterprise) == TRUE)) { // // Enforce the Advanced Server physical memory limit. // On PAE machines the system continues to operate in 8-byte PTE mode. // TotalPagesAllowed = MI_ADS_MAX_PAGES; } else if (ExVerifySuite(Blade) == TRUE) { // // Enforce the Blade physical memory limit. // TotalPagesAllowed = MI_BLADE_MAX_PAGES; } // // Walk through the memory descriptors and remove or truncate descriptors // that exceed the maximum physical memory to be used. // PageCount = 0; NextMd = LoaderBlock->MemoryDescriptorListHead.Flink; while (NextMd != &LoaderBlock->MemoryDescriptorListHead) { MemoryDescriptor = CONTAINING_RECORD(NextMd, MEMORY_ALLOCATION_DESCRIPTOR, ListEntry); if ((MemoryDescriptor->MemoryType == LoaderFirmwarePermanent) || (MemoryDescriptor->MemoryType == LoaderBBTMemory) || (MemoryDescriptor->MemoryType == LoaderBad) || (MemoryDescriptor->MemoryType == LoaderSpecialMemory)) { NextMd = MemoryDescriptor->ListEntry.Flink; continue; } PageCount += MemoryDescriptor->PageCount; if (PageCount <= TotalPagesAllowed) { NextMd = MemoryDescriptor->ListEntry.Flink; continue; } // // This descriptor needs to be removed or truncated. // if (PageCount - MemoryDescriptor->PageCount >= TotalPagesAllowed) { // // Completely remove this descriptor. // // Note since this only adjusts the links and since the entry is // not freed, it can still be safely referenced again below to // obtain the NextMd. N.B. This keeps the memory descriptors // sorted in ascending order. // RemoveEntryList (NextMd); } else { // // Truncate this descriptor. // ASSERT (PageCount - MemoryDescriptor->PageCount < TotalPagesAllowed); MemoryDescriptor->PageCount -= (ULONG)(PageCount - TotalPagesAllowed); PageCount = TotalPagesAllowed; } NextMd = MemoryDescriptor->ListEntry.Flink; } return; } VOID MmFreeLoaderBlock ( IN PLOADER_PARAMETER_BLOCK LoaderBlock ) /*++ Routine Description: This function is called as the last routine in phase 1 initialization. It frees memory used by the OsLoader. Arguments: LoaderBlock - Supplies a pointer to the system loader block. Return Value: None. Environment: Kernel Mode Only. System initialization. --*/ { PLIST_ENTRY NextMd; PMEMORY_ALLOCATION_DESCRIPTOR MemoryDescriptor; ULONG i; PFN_NUMBER NextPhysicalPage; PFN_NUMBER PagesFreed; PMMPFN Pfn1; KIRQL OldIrql; PPHYSICAL_MEMORY_RUN RunBase; PPHYSICAL_MEMORY_RUN Runs; i = 0; NextMd = LoaderBlock->MemoryDescriptorListHead.Flink; while (NextMd != &LoaderBlock->MemoryDescriptorListHead) { i += 1; MemoryDescriptor = CONTAINING_RECORD(NextMd, MEMORY_ALLOCATION_DESCRIPTOR, ListEntry); NextMd = MemoryDescriptor->ListEntry.Flink; } RunBase = ExAllocatePoolWithTag (NonPagedPool, sizeof(PHYSICAL_MEMORY_RUN) * i, 'lMmM'); if (RunBase == NULL) { return; } Runs = RunBase; // // // Walk through the memory descriptors and add pages to the // free list in the PFN database. // NextMd = LoaderBlock->MemoryDescriptorListHead.Flink; while (NextMd != &LoaderBlock->MemoryDescriptorListHead) { MemoryDescriptor = CONTAINING_RECORD(NextMd, MEMORY_ALLOCATION_DESCRIPTOR, ListEntry); switch (MemoryDescriptor->MemoryType) { case LoaderOsloaderHeap: case LoaderRegistryData: case LoaderNlsData: //case LoaderMemoryData: //this has page table and other stuff. // // Capture the data to temporary storage so we won't // free memory we are referencing. // Runs->BasePage = MemoryDescriptor->BasePage; Runs->PageCount = MemoryDescriptor->PageCount; Runs += 1; break; default: break; } NextMd = MemoryDescriptor->ListEntry.Flink; } PagesFreed = 0; LOCK_PFN (OldIrql); if (Runs != RunBase) { Runs -= 1; do { i = (ULONG)Runs->PageCount; NextPhysicalPage = Runs->BasePage; #if defined (_MI_MORE_THAN_4GB_) if (MiNoLowMemory != 0) { if (NextPhysicalPage < MiNoLowMemory) { // // Don't free this run as it is below the memory threshold // configured for this system. // Runs -= 1; continue; } } #endif Pfn1 = MI_PFN_ELEMENT (NextPhysicalPage); PagesFreed += i; while (i != 0) { if (Pfn1->u3.e2.ReferenceCount == 0) { if (Pfn1->u1.Flink == 0) { // // Set the PTE address to the physical page for // virtual address alignment checking. // Pfn1->PteAddress = (PMMPTE)(NextPhysicalPage << PTE_SHIFT); MiDetermineNode (NextPhysicalPage, Pfn1); MiInsertPageInFreeList (NextPhysicalPage); } } else { if (NextPhysicalPage != 0) { // // Remove PTE and insert into the free list. If it is // a physical address within the PFN database, the PTE // element does not exist and therefore cannot be updated. // if (!MI_IS_PHYSICAL_ADDRESS ( MiGetVirtualAddressMappedByPte (Pfn1->PteAddress))) { // // Not a physical address. // *(Pfn1->PteAddress) = ZeroPte; } MI_SET_PFN_DELETED (Pfn1); MiDecrementShareCountOnly (NextPhysicalPage); } } Pfn1 += 1; i -= 1; NextPhysicalPage += 1; } Runs -= 1; } while (Runs >= RunBase); } // // Since systemwide commitment was determined early in Phase 0 and // excluded the ranges just freed, add them back in now. // if (PagesFreed != 0) { InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, PagesFreed); InterlockedExchangeAddSizeT (&MmTotalCommitLimit, PagesFreed); } #if defined(_X86_) if (MmVirtualBias != 0) { // // If the kernel has been biased to allow for 3gb of user address space, // then the first 16mb of memory is doubly mapped to KSEG0_BASE and to // ALTERNATE_BASE. Therefore, the KSEG0_BASE entries must be unmapped. // PMMPTE Pde; ULONG NumberOfPdes; NumberOfPdes = MmBootImageSize / MM_VA_MAPPED_BY_PDE; Pde = MiGetPdeAddress((PVOID)KSEG0_BASE); for (i = 0; i < NumberOfPdes; i += 1) { MI_WRITE_INVALID_PTE (Pde, ZeroKernelPte); Pde += 1; } } #endif KeFlushEntireTb (TRUE, TRUE); UNLOCK_PFN (OldIrql); ExFreePool (RunBase); return; } VOID MiBuildPagedPool ( VOID ) /*++ Routine Description: This function is called to build the structures required for paged pool and initialize the pool. Once this routine is called, paged pool may be allocated. Arguments: None. Return Value: None. Environment: Kernel Mode Only. System initialization. --*/ { SIZE_T Size; PMMPTE PointerPte; PMMPTE LastPte; PMMPTE LastPde; PMMPTE PointerPde; MMPTE TempPte; PFN_NUMBER PageFrameIndex; PFN_NUMBER ContainingFrame; SIZE_T AdditionalCommittedPages; KIRQL OldIrql; ULONG i; #if (_MI_PAGING_LEVELS >= 4) PMMPTE PointerPxe; PMMPTE PointerPxeEnd; #endif #if (_MI_PAGING_LEVELS >= 3) PVOID LastVa; PMMPTE PointerPpe; PMMPTE PointerPpeEnd; #else PMMPFN Pfn1; #endif i = 0; AdditionalCommittedPages = 0; #if (_MI_PAGING_LEVELS < 3) // // Double map system page directory page. // PointerPte = MiGetPteAddress(PDE_BASE); for (i = 0 ; i < PD_PER_SYSTEM; i += 1) { MmSystemPageDirectory[i] = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte); Pfn1 = MI_PFN_ELEMENT(MmSystemPageDirectory[i]); Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE; PointerPte += 1; } // // Was not mapped physically, map it virtually in system space. // PointerPte = MiReserveSystemPtes (PD_PER_SYSTEM, SystemPteSpace); if (PointerPte == NULL) { MiIssueNoPtesBugcheck (PD_PER_SYSTEM, SystemPteSpace); } MmSystemPagePtes = (PMMPTE)MiGetVirtualAddressMappedByPte (PointerPte); TempPte = ValidKernelPde; for (i = 0 ; i < PD_PER_SYSTEM; i += 1) { TempPte.u.Hard.PageFrameNumber = MmSystemPageDirectory[i]; MI_WRITE_VALID_PTE (PointerPte, TempPte); PointerPte += 1; } #endif if (MmPagedPoolMaximumDesired == TRUE) { MmSizeOfPagedPoolInBytes = ((PCHAR)MmNonPagedSystemStart - (PCHAR)MmPagedPoolStart); } else if (MmSizeOfPagedPoolInBytes == 0) { // // A size of 0 means size the pool based on physical memory. // MmSizeOfPagedPoolInBytes = 2 * MmMaximumNonPagedPoolInBytes; #if (_MI_PAGING_LEVELS >= 3) MmSizeOfPagedPoolInBytes *= 2; #endif } if (MmIsThisAnNtAsSystem()) { if ((MmNumberOfPhysicalPages > ((24*1024*1024) >> PAGE_SHIFT)) && (MmSizeOfPagedPoolInBytes < MM_MINIMUM_PAGED_POOL_NTAS)) { MmSizeOfPagedPoolInBytes = MM_MINIMUM_PAGED_POOL_NTAS; } } if (MmSizeOfPagedPoolInBytes > (ULONG_PTR)((PCHAR)MmNonPagedSystemStart - (PCHAR)MmPagedPoolStart)) { MmSizeOfPagedPoolInBytes = ((PCHAR)MmNonPagedSystemStart - (PCHAR)MmPagedPoolStart); } Size = BYTES_TO_PAGES(MmSizeOfPagedPoolInBytes); if (Size < MM_MIN_INITIAL_PAGED_POOL) { Size = MM_MIN_INITIAL_PAGED_POOL; } if (Size > (MM_MAX_PAGED_POOL >> PAGE_SHIFT)) { Size = MM_MAX_PAGED_POOL >> PAGE_SHIFT; } #if defined (_WIN64) // // NT64 places system mapped views directly after paged pool. Ensure // enough VA space is available. // if (Size + (MmSystemViewSize >> PAGE_SHIFT) > (MM_MAX_PAGED_POOL >> PAGE_SHIFT)) { ASSERT (MmSizeOfPagedPoolInBytes > 2 * MmSystemViewSize); MmSizeOfPagedPoolInBytes -= MmSystemViewSize; Size = BYTES_TO_PAGES(MmSizeOfPagedPoolInBytes); } #endif Size = (Size + (PTE_PER_PAGE - 1)) / PTE_PER_PAGE; MmSizeOfPagedPoolInBytes = (ULONG_PTR)Size * PAGE_SIZE * PTE_PER_PAGE; // // Set size to the number of pages in the pool. // Size = Size * PTE_PER_PAGE; // // If paged pool is really nonpagable then limit the size based // on how much physical memory is actually present. Disable this // feature if not enough physical memory is present to do it. // if (MmDisablePagingExecutive & MM_PAGED_POOL_LOCKED_DOWN) { Size = MmSizeOfPagedPoolInBytes / PAGE_SIZE; if ((MI_NONPAGABLE_MEMORY_AVAILABLE() < 2048) || (MmAvailablePages < 2048)) { Size = 0; } else { if ((SPFN_NUMBER)(Size) > MI_NONPAGABLE_MEMORY_AVAILABLE() - 2048) { Size = (MI_NONPAGABLE_MEMORY_AVAILABLE() - 2048); } if (Size > MmAvailablePages - 2048) { Size = MmAvailablePages - 2048; } } Size = ((Size * PAGE_SIZE) / MM_VA_MAPPED_BY_PDE) * MM_VA_MAPPED_BY_PDE; if ((((Size / 5) * 4) >= MmSizeOfPagedPoolInBytes) && (Size >= MM_MIN_INITIAL_PAGED_POOL)) { MmSizeOfPagedPoolInBytes = Size; } else { MmDisablePagingExecutive &= ~MM_PAGED_POOL_LOCKED_DOWN; } Size = MmSizeOfPagedPoolInBytes >> PAGE_SHIFT; } ASSERT ((MmSizeOfPagedPoolInBytes + (PCHAR)MmPagedPoolStart) <= (PCHAR)MmNonPagedSystemStart); ASSERT64 ((MmSizeOfPagedPoolInBytes + (PCHAR)MmPagedPoolStart + MmSystemViewSize) <= (PCHAR)MmNonPagedSystemStart); MmPagedPoolEnd = (PVOID)(((PUCHAR)MmPagedPoolStart + MmSizeOfPagedPoolInBytes) - 1); MmPageAlignedPoolBase[PagedPool] = MmPagedPoolStart; // // Build page table page for paged pool. // PointerPde = MiGetPdeAddress (MmPagedPoolStart); TempPte = ValidKernelPde; #if (_MI_PAGING_LEVELS >= 3) // // Map in all the page directory pages to span all of paged pool. // This removes the need for a system lookup directory. // LastVa = (PVOID)((PCHAR)MmPagedPoolEnd + MmSystemViewSize); PointerPpe = MiGetPpeAddress (MmPagedPoolStart); PointerPpeEnd = MiGetPpeAddress (LastVa); MiSystemViewStart = (ULONG_PTR)MmPagedPoolEnd + 1; PointerPde = MiGetPdeAddress (MmPagedPoolEnd) + 1; LastPde = MiGetPdeAddress (LastVa); LOCK_PFN (OldIrql); #if (_MI_PAGING_LEVELS >= 4) PointerPxe = MiGetPxeAddress (MmPagedPoolStart); PointerPxeEnd = MiGetPxeAddress (LastVa); while (PointerPxe <= PointerPxeEnd) { if (PointerPxe->u.Hard.Valid == 0) { PageFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (PointerPxe)); TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (PointerPxe, TempPte); MiInitializePfn (PageFrameIndex, PointerPxe, 1); // // Make all entries no access since the PDEs may not fill the page. // MiFillMemoryPte (MiGetVirtualAddressMappedByPte (PointerPxe), PAGE_SIZE, MM_KERNEL_NOACCESS_PTE); MmResidentAvailablePages -= 1; AdditionalCommittedPages += 1; } PointerPxe += 1; } #endif while (PointerPpe <= PointerPpeEnd) { if (PointerPpe->u.Hard.Valid == 0) { PageFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (PointerPpe)); TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (PointerPpe, TempPte); MiInitializePfn (PageFrameIndex, PointerPpe, 1); // // Make all entries no access since the PDEs may not fill the page. // MiFillMemoryPte (MiGetVirtualAddressMappedByPte (PointerPpe), PAGE_SIZE, MM_KERNEL_NOACCESS_PTE); MmResidentAvailablePages -= 1; AdditionalCommittedPages += 1; } PointerPpe += 1; } // // Initialize the system view page table pages. // MmResidentAvailablePages -= (LastPde - PointerPde + 1); AdditionalCommittedPages += (LastPde - PointerPde + 1); while (PointerPde <= LastPde) { ASSERT (PointerPde->u.Hard.Valid == 0); PageFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (PointerPde)); TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (PointerPde, TempPte); MiInitializePfn (PageFrameIndex, PointerPde, 1); MiFillMemoryPte (MiGetVirtualAddressMappedByPte (PointerPde), PAGE_SIZE, ZeroKernelPte.u.Long); PointerPde += 1; } UNLOCK_PFN (OldIrql); PointerPde = MiGetPdeAddress (MmPagedPoolStart); #endif PointerPte = MiGetPteAddress (MmPagedPoolStart); MmPagedPoolInfo.FirstPteForPagedPool = PointerPte; MmPagedPoolInfo.LastPteForPagedPool = MiGetPteAddress (MmPagedPoolEnd); MiFillMemoryPte (PointerPde, sizeof(MMPTE) * (1 + MiGetPdeAddress (MmPagedPoolEnd) - PointerPde), MM_KERNEL_NOACCESS_PTE); LOCK_PFN (OldIrql); // // Map in a page table page. // PageFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (PointerPde)); TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (PointerPde, TempPte); #if (_MI_PAGING_LEVELS >= 3) ContainingFrame = MI_GET_PAGE_FRAME_FROM_PTE(MiGetPpeAddress (MmPagedPoolStart)); #else ContainingFrame = MmSystemPageDirectory[(PointerPde - MiGetPdeAddress(0)) / PDE_PER_PAGE]; #endif MiInitializePfnForOtherProcess (PageFrameIndex, PointerPde, ContainingFrame); MiFillMemoryPte (PointerPte, PAGE_SIZE, MM_KERNEL_NOACCESS_PTE); UNLOCK_PFN (OldIrql); MmPagedPoolInfo.NextPdeForPagedPoolExpansion = PointerPde + 1; // // Build bitmaps for paged pool. // MiCreateBitMap (&MmPagedPoolInfo.PagedPoolAllocationMap, Size, NonPagedPool); RtlSetAllBits (MmPagedPoolInfo.PagedPoolAllocationMap); // // Indicate first page worth of PTEs are available. // RtlClearBits (MmPagedPoolInfo.PagedPoolAllocationMap, 0, PTE_PER_PAGE); MiCreateBitMap (&MmPagedPoolInfo.EndOfPagedPoolBitmap, Size, NonPagedPool); RtlClearAllBits (MmPagedPoolInfo.EndOfPagedPoolBitmap); // // If verifier is present then build the verifier paged pool bitmap. // if (MmVerifyDriverBufferLength != (ULONG)-1) { MiCreateBitMap (&VerifierLargePagedPoolMap, Size, NonPagedPool); RtlClearAllBits (VerifierLargePagedPoolMap); } // // Initialize paged pool. // InitializePool (PagedPool, 0L); // // If paged pool is really nonpagable then allocate the memory now. // if (MmDisablePagingExecutive & MM_PAGED_POOL_LOCKED_DOWN) { PointerPde = MiGetPdeAddress (MmPagedPoolStart); PointerPde += 1; LastPde = MiGetPdeAddress (MmPagedPoolEnd); TempPte = ValidKernelPde; PointerPte = MiGetPteAddress (MmPagedPoolStart); LastPte = MiGetPteAddress (MmPagedPoolEnd); ASSERT (MmPagedPoolCommit == 0); MmPagedPoolCommit = (ULONG)(LastPte - PointerPte + 1); ASSERT (MmPagedPoolInfo.PagedPoolCommit == 0); MmPagedPoolInfo.PagedPoolCommit = MmPagedPoolCommit; #if DBG // // Ensure no paged pool has been allocated yet. // for (i = 0; i < PTE_PER_PAGE; i += 1) { ASSERT (!RtlCheckBit (MmPagedPoolInfo.PagedPoolAllocationMap, i)); } while (i < MmSizeOfPagedPoolInBytes / PAGE_SIZE) { ASSERT (RtlCheckBit (MmPagedPoolInfo.PagedPoolAllocationMap, i)); i += 1; } #endif RtlClearAllBits (MmPagedPoolInfo.PagedPoolAllocationMap); LOCK_PFN (OldIrql); // // Map in the page table pages. // MmResidentAvailablePages -= (LastPde - PointerPde + 1); AdditionalCommittedPages += (LastPde - PointerPde + 1); while (PointerPde <= LastPde) { ASSERT (PointerPde->u.Hard.Valid == 0); PageFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (PointerPde)); TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (PointerPde, TempPte); #if (_MI_PAGING_LEVELS >= 3) ContainingFrame = MI_GET_PAGE_FRAME_FROM_PTE(MiGetPteAddress (PointerPde)); #else ContainingFrame = MmSystemPageDirectory[(PointerPde - MiGetPdeAddress(0)) / PDE_PER_PAGE]; #endif MiInitializePfnForOtherProcess (PageFrameIndex, MiGetPteAddress (PointerPde), ContainingFrame); MiFillMemoryPte (MiGetVirtualAddressMappedByPte (PointerPde), PAGE_SIZE, MM_KERNEL_NOACCESS_PTE); PointerPde += 1; } MmPagedPoolInfo.NextPdeForPagedPoolExpansion = PointerPde; TempPte = ValidKernelPte; MI_SET_PTE_DIRTY (TempPte); ASSERT (MmAvailablePages > (PFN_COUNT)(LastPte - PointerPte + 1)); ASSERT (MmResidentAvailablePages > (SPFN_NUMBER)(LastPte - PointerPte + 1)); MmResidentAvailablePages -= (LastPte - PointerPte + 1); AdditionalCommittedPages += (LastPte - PointerPte + 1); while (PointerPte <= LastPte) { ASSERT (PointerPte->u.Hard.Valid == 0); PageFrameIndex = MiRemoveAnyPage( MI_GET_PAGE_COLOR_FROM_PTE (PointerPte)); TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (PointerPte, TempPte); MiInitializePfn (PageFrameIndex, PointerPte, 1); PointerPte += 1; } UNLOCK_PFN (OldIrql); } // // Since the commitment return path is lock free, the total committed // page count must be atomically incremented. // InterlockedExchangeAddSizeT (&MmTotalCommittedPages, AdditionalCommittedPages); MiInitializeSpecialPool (NonPagedPool); // // Allow mapping of views into system space. // MiInitializeSystemSpaceMap (NULL); return; } VOID MiFindInitializationCode ( OUT PVOID *StartVa, OUT PVOID *EndVa ) /*++ Routine Description: This function locates the start and end of the kernel initialization code. This code resides in the "init" section of the kernel image. Arguments: StartVa - Returns the starting address of the init section. EndVa - Returns the ending address of the init section. Return Value: None. Environment: Kernel Mode Only. End of system initialization. --*/ { PKLDR_DATA_TABLE_ENTRY LdrDataTableEntry; PVOID CurrentBase; PVOID InitStart; PVOID InitEnd; PLIST_ENTRY Next; PIMAGE_NT_HEADERS NtHeader; PIMAGE_SECTION_HEADER SectionTableEntry; PIMAGE_SECTION_HEADER LastDiscard; LONG i; LOGICAL DiscardSection; PVOID MiFindInitializationCodeAddress; PKTHREAD CurrentThread; MiFindInitializationCodeAddress = MmGetProcedureAddress((PVOID)(ULONG_PTR)&MiFindInitializationCode); #if defined(_IA64_) // // One more indirection is needed due to the PLABEL. // MiFindInitializationCodeAddress = (PVOID)(*((PULONGLONG)MiFindInitializationCodeAddress)); #endif *StartVa = NULL; // // Walk through the loader blocks looking for the base which // contains this routine. // CurrentThread = KeGetCurrentThread (); KeEnterCriticalRegionThread (CurrentThread); ExAcquireResourceExclusiveLite (&PsLoadedModuleResource, TRUE); Next = PsLoadedModuleList.Flink; while (Next != &PsLoadedModuleList) { LdrDataTableEntry = CONTAINING_RECORD (Next, KLDR_DATA_TABLE_ENTRY, InLoadOrderLinks); if (LdrDataTableEntry->SectionPointer != NULL) { // // This entry was loaded by MmLoadSystemImage so it's already // had its init section removed. // Next = Next->Flink; continue; } CurrentBase = (PVOID)LdrDataTableEntry->DllBase; NtHeader = RtlImageNtHeader(CurrentBase); SectionTableEntry = (PIMAGE_SECTION_HEADER)((PCHAR)NtHeader + sizeof(ULONG) + sizeof(IMAGE_FILE_HEADER) + NtHeader->FileHeader.SizeOfOptionalHeader); // // From the image header, locate the sections named 'INIT', // PAGEVRF* and PAGESPEC. INIT always goes, the others go depending // on registry configuration. // i = NtHeader->FileHeader.NumberOfSections; InitStart = NULL; while (i > 0) { #if DBG if ((*(PULONG)SectionTableEntry->Name == 'tini') || (*(PULONG)SectionTableEntry->Name == 'egap')) { DbgPrint("driver %wZ has lower case sections (init or pagexxx)\n", &LdrDataTableEntry->FullDllName); } #endif DiscardSection = FALSE; // // Free any INIT sections (or relocation sections that haven't // been already). Note a driver may have a relocation section // but not have any INIT code. // if ((*(PULONG)SectionTableEntry->Name == 'TINI') || ((SectionTableEntry->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) != 0)) { DiscardSection = TRUE; } else if ((*(PULONG)SectionTableEntry->Name == 'EGAP') && (SectionTableEntry->Name[4] == 'V') && (SectionTableEntry->Name[5] == 'R') && (SectionTableEntry->Name[6] == 'F')) { // // Discard PAGEVRF* if no drivers are being instrumented. // if (MmVerifyDriverBufferLength == (ULONG)-1) { DiscardSection = TRUE; } } else if ((*(PULONG)SectionTableEntry->Name == 'EGAP') && (*(PULONG)&SectionTableEntry->Name[4] == 'CEPS')) { // // Discard PAGESPEC special pool code if it's not enabled. // if (MiSpecialPoolFirstPte == NULL) { DiscardSection = TRUE; } } if (DiscardSection == TRUE) { InitStart = (PVOID)((PCHAR)CurrentBase + SectionTableEntry->VirtualAddress); InitEnd = (PVOID)((PCHAR)InitStart + SectionTableEntry->SizeOfRawData - 1); InitEnd = (PVOID)((PCHAR)PAGE_ALIGN ((PCHAR)InitEnd + (NtHeader->OptionalHeader.SectionAlignment - 1)) - 1); InitStart = (PVOID)ROUND_TO_PAGES (InitStart); // // Check if more sections are discardable after this one so // even small INIT sections can be discarded. // if (i == 1) { LastDiscard = SectionTableEntry; } else { LastDiscard = NULL; do { i -= 1; SectionTableEntry += 1; if ((SectionTableEntry->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) != 0) { // // Discard this too. // LastDiscard = SectionTableEntry; } else { break; } } while (i > 1); } if (LastDiscard) { InitEnd = (PVOID)(((PCHAR)CurrentBase + LastDiscard->VirtualAddress) + (LastDiscard->SizeOfRawData - 1)); // // If this isn't the last section in the driver then the // the next section is not discardable. So the last // section is not rounded down, but all others must be. // if (i != 1) { InitEnd = (PVOID)((PCHAR)PAGE_ALIGN ((PCHAR)InitEnd + (NtHeader->OptionalHeader.SectionAlignment - 1)) - 1); } } if (InitEnd > (PVOID)((PCHAR)CurrentBase + LdrDataTableEntry->SizeOfImage)) { InitEnd = (PVOID)(((ULONG_PTR)CurrentBase + (LdrDataTableEntry->SizeOfImage - 1)) | (PAGE_SIZE - 1)); } if (InitStart <= InitEnd) { if ((MiFindInitializationCodeAddress >= InitStart) && (MiFindInitializationCodeAddress <= InitEnd)) { // // This init section is in the kernel, don't free it // now as it would free this code! // ASSERT (*StartVa == NULL); *StartVa = InitStart; *EndVa = InitEnd; } else { MiFreeInitializationCode (InitStart, InitEnd); } } } i -= 1; SectionTableEntry += 1; } Next = Next->Flink; } ExReleaseResourceLite (&PsLoadedModuleResource); KeLeaveCriticalRegionThread (CurrentThread); return; } VOID MiFreeInitializationCode ( IN PVOID StartVa, IN PVOID EndVa ) /*++ Routine Description: This function is called to delete the initialization code. Arguments: StartVa - Supplies the starting address of the range to delete. EndVa - Supplies the ending address of the range to delete. Return Value: None. Environment: Kernel Mode Only. Runs after system initialization. --*/ { PMMPFN Pfn1; PMMPTE PointerPte; PFN_NUMBER PageFrameIndex; PFN_NUMBER PagesFreed; KIRQL OldIrql; ASSERT(ExPageLockHandle); #if defined (_MI_MORE_THAN_4GB_) if (MiNoLowMemory != 0) { // // Don't free this range as the kernel is always below the memory // threshold configured for this system. // return; } #endif PagesFreed = 0; MmLockPagableSectionByHandle(ExPageLockHandle); if (MI_IS_PHYSICAL_ADDRESS(StartVa)) { LOCK_PFN (OldIrql); while (StartVa < EndVa) { // // On certain architectures (e.g., MIPS) virtual addresses // may be physical and hence have no corresponding PTE. // PageFrameIndex = MI_CONVERT_PHYSICAL_TO_PFN (StartVa); Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); Pfn1->u2.ShareCount = 0; Pfn1->u3.e2.ReferenceCount = 0; MI_SET_PFN_DELETED (Pfn1); MiInsertPageInFreeList (PageFrameIndex); StartVa = (PVOID)((PUCHAR)StartVa + PAGE_SIZE); PagesFreed += 1; } UNLOCK_PFN (OldIrql); } else { PointerPte = MiGetPteAddress (StartVa); PagesFreed = MiDeleteSystemPagableVm (PointerPte, (PFN_NUMBER) (1 + MiGetPteAddress (EndVa) - PointerPte), ZeroKernelPte, FALSE, NULL); } MmUnlockPagableImageSection(ExPageLockHandle); // // Since systemwide commitment was determined early in Phase 0 and // excluded the ranges just freed, add them back in now. // if (PagesFreed != 0) { // // Since systemwide commitment was determined early in Phase 0 // and excluded the ranges just freed, increase the limits // accordingly now. Note that there is no commitment to be // returned (as none was ever charged earlier) for boot // loaded drivers. // InterlockedExchangeAddSizeT (&MmTotalCommitLimitMaximum, PagesFreed); InterlockedExchangeAddSizeT (&MmTotalCommitLimit, PagesFreed); } return; } VOID MiEnablePagingTheExecutive ( VOID ) /*++ Routine Description: This function locates the start and end of the kernel initialization code. This code resides in the "init" section of the kernel image. Arguments: StartVa - Returns the starting address of the init section. EndVa - Returns the ending address of the init section. Return Value: None. Environment: Kernel Mode Only. End of system initialization. --*/ { KIRQL OldIrql; KIRQL OldIrqlWs; PVOID StartVa; PETHREAD CurrentThread; PLONG SectionLockCountPointer; PKLDR_DATA_TABLE_ENTRY LdrDataTableEntry; PVOID CurrentBase; PLIST_ENTRY Next; PIMAGE_NT_HEADERS NtHeader; PIMAGE_SECTION_HEADER StartSectionTableEntry; PIMAGE_SECTION_HEADER SectionTableEntry; LONG i; PMMPTE PointerPte; PMMPTE LastPte; PMMPTE SubsectionStartPte; PMMPTE SubsectionLastPte; LOGICAL PageSection; PVOID SectionBaseAddress; LOGICAL AlreadyLockedOnce; ULONG Waited; PEPROCESS CurrentProcess; // // Don't page kernel mode code if customer does not want it paged or if // this is a diskless remote boot client. // if (MmDisablePagingExecutive & MM_SYSTEM_CODE_LOCKED_DOWN) { return; } #if defined(REMOTE_BOOT) if (IoRemoteBootClient && IoCscInitializationFailed) { return; } #endif // // Initializing LastPte is not needed for correctness, but // without it the compiler cannot compile this code W4 to check // for use of uninitialized variables. // LastPte = NULL; // // Walk through the loader blocks looking for the base which // contains this routine. // CurrentThread = PsGetCurrentThread (); CurrentProcess = PsGetCurrentProcessByThread (CurrentThread); KeEnterCriticalRegionThread (&CurrentThread->Tcb); ExAcquireResourceExclusiveLite (&PsLoadedModuleResource, TRUE); Next = PsLoadedModuleList.Flink; while (Next != &PsLoadedModuleList) { LdrDataTableEntry = CONTAINING_RECORD (Next, KLDR_DATA_TABLE_ENTRY, InLoadOrderLinks); if (LdrDataTableEntry->SectionPointer != NULL) { // // This entry was loaded by MmLoadSystemImage so it's already paged. // Next = Next->Flink; continue; } CurrentBase = (PVOID)LdrDataTableEntry->DllBase; if (MI_IS_PHYSICAL_ADDRESS (CurrentBase)) { // // Mapped physically, can't be paged. // Next = Next->Flink; continue; } NtHeader = RtlImageNtHeader (CurrentBase); restart: StartSectionTableEntry = NULL; SectionTableEntry = (PIMAGE_SECTION_HEADER)((PCHAR)NtHeader + sizeof(ULONG) + sizeof(IMAGE_FILE_HEADER) + NtHeader->FileHeader.SizeOfOptionalHeader); // // From the image header, locate the section named 'PAGE' or '.edata'. // i = NtHeader->FileHeader.NumberOfSections; PointerPte = NULL; while (i > 0) { SectionBaseAddress = SECTION_BASE_ADDRESS(SectionTableEntry); if ((PUCHAR)SectionBaseAddress == ((PUCHAR)CurrentBase + SectionTableEntry->VirtualAddress)) { AlreadyLockedOnce = TRUE; // // This subsection has already been locked down (and possibly // unlocked as well) at least once. If it is NOT locked down // right now and the pages are not in the system working set // then include it in the chunk to be paged. // SectionLockCountPointer = SECTION_LOCK_COUNT_POINTER (SectionTableEntry); if (*SectionLockCountPointer == 0) { SubsectionStartPte = MiGetPteAddress ((PVOID)(ROUND_TO_PAGES ( (ULONG_PTR)CurrentBase + SectionTableEntry->VirtualAddress))); SubsectionLastPte = MiGetPteAddress ((PVOID)((ULONG_PTR)CurrentBase + SectionTableEntry->VirtualAddress + (NtHeader->OptionalHeader.SectionAlignment - 1) + SectionTableEntry->SizeOfRawData - PAGE_SIZE)); if (SubsectionLastPte >= SubsectionStartPte) { AlreadyLockedOnce = FALSE; } } } else { AlreadyLockedOnce = FALSE; } PageSection = ((*(PULONG)SectionTableEntry->Name == 'EGAP') || (*(PULONG)SectionTableEntry->Name == 'ade.')) && (AlreadyLockedOnce == FALSE); if (*(PULONG)SectionTableEntry->Name == 'EGAP' && SectionTableEntry->Name[4] == 'K' && SectionTableEntry->Name[5] == 'D') { // // Only pageout PAGEKD if KdPitchDebugger is TRUE. // PageSection = KdPitchDebugger; } if ((*(PULONG)SectionTableEntry->Name == 'EGAP') && (SectionTableEntry->Name[4] == 'V') && (SectionTableEntry->Name[5] == 'R') && (SectionTableEntry->Name[6] == 'F')) { // // Pageout PAGEVRF* if no drivers are being instrumented. // if (MmVerifyDriverBufferLength != (ULONG)-1) { PageSection = FALSE; } } if ((*(PULONG)SectionTableEntry->Name == 'EGAP') && (*(PULONG)&SectionTableEntry->Name[4] == 'CEPS')) { // // Pageout PAGESPEC special pool code if it's not enabled. // if (MiSpecialPoolFirstPte != NULL) { PageSection = FALSE; } } if (PageSection) { // // This section is pagable, save away the start and end. // if (PointerPte == NULL) { // // Previous section was NOT pagable, get the start address. // ASSERT (StartSectionTableEntry == NULL); StartSectionTableEntry = SectionTableEntry; PointerPte = MiGetPteAddress ((PVOID)(ROUND_TO_PAGES ( (ULONG_PTR)CurrentBase + SectionTableEntry->VirtualAddress))); } LastPte = MiGetPteAddress ((PVOID)((ULONG_PTR)CurrentBase + SectionTableEntry->VirtualAddress + (NtHeader->OptionalHeader.SectionAlignment - 1) + SectionTableEntry->SizeOfRawData - PAGE_SIZE)); } else { // // This section is not pagable, if the previous section was // pagable, enable it. // if (PointerPte != NULL) { ASSERT (StartSectionTableEntry != NULL); LOCK_SYSTEM_WS (OldIrqlWs, CurrentThread); LOCK_PFN (OldIrql); StartVa = PAGE_ALIGN (StartSectionTableEntry); while (StartVa < (PVOID) SectionTableEntry) { Waited = MiMakeSystemAddressValidPfnSystemWs (StartVa); if (Waited != 0) { // // Restart at the top as the locks were released. // UNLOCK_PFN (OldIrql); UNLOCK_SYSTEM_WS (OldIrqlWs); goto restart; } StartVa = (PVOID)((PCHAR)StartVa + PAGE_SIZE); } // // Now that we're holding the proper locks, rewalk all // the sections to make sure they weren't locked down // after we checked above. // while (StartSectionTableEntry < SectionTableEntry) { SectionBaseAddress = SECTION_BASE_ADDRESS(StartSectionTableEntry); SectionLockCountPointer = SECTION_LOCK_COUNT_POINTER (StartSectionTableEntry); if (((PUCHAR)SectionBaseAddress == ((PUCHAR)CurrentBase + StartSectionTableEntry->VirtualAddress)) && (*SectionLockCountPointer != 0)) { // // Restart at the top as the section has been // explicitly locked by a driver since we first // checked above. // UNLOCK_PFN (OldIrql); UNLOCK_SYSTEM_WS (OldIrqlWs); goto restart; } StartSectionTableEntry += 1; } MiEnablePagingOfDriverAtInit (PointerPte, LastPte); UNLOCK_PFN (OldIrql); UNLOCK_SYSTEM_WS (OldIrqlWs); PointerPte = NULL; StartSectionTableEntry = NULL; } } i -= 1; SectionTableEntry += 1; } if (PointerPte != NULL) { ASSERT (StartSectionTableEntry != NULL); LOCK_SYSTEM_WS (OldIrqlWs, CurrentThread); LOCK_PFN (OldIrql); StartVa = PAGE_ALIGN (StartSectionTableEntry); while (StartVa < (PVOID) SectionTableEntry) { Waited = MiMakeSystemAddressValidPfnSystemWs (StartVa); if (Waited != 0) { // // Restart at the top as the locks were released. // UNLOCK_PFN (OldIrql); UNLOCK_SYSTEM_WS (OldIrqlWs); goto restart; } StartVa = (PVOID)((PCHAR)StartVa + PAGE_SIZE); } // // Now that we're holding the proper locks, rewalk all // the sections to make sure they weren't locked down // after we checked above. // while (StartSectionTableEntry < SectionTableEntry) { SectionBaseAddress = SECTION_BASE_ADDRESS(StartSectionTableEntry); SectionLockCountPointer = SECTION_LOCK_COUNT_POINTER (StartSectionTableEntry); if (((PUCHAR)SectionBaseAddress == ((PUCHAR)CurrentBase + StartSectionTableEntry->VirtualAddress)) && (*SectionLockCountPointer != 0)) { // // Restart at the top as the section has been // explicitly locked by a driver since we first // checked above. // UNLOCK_PFN (OldIrql); UNLOCK_SYSTEM_WS (OldIrqlWs); goto restart; } StartSectionTableEntry += 1; } MiEnablePagingOfDriverAtInit (PointerPte, LastPte); UNLOCK_PFN (OldIrql); UNLOCK_SYSTEM_WS (OldIrqlWs); } Next = Next->Flink; } ExReleaseResourceLite (&PsLoadedModuleResource); KeLeaveCriticalRegionThread (&CurrentThread->Tcb); return; } VOID MiEnablePagingOfDriverAtInit ( IN PMMPTE PointerPte, IN PMMPTE LastPte ) /*++ Routine Description: This routine marks the specified range of PTEs as pagable. Arguments: PointerPte - Supplies the starting PTE. LastPte - Supplies the ending PTE. Return Value: None. Environment: Working set mutex AND PFN lock held. --*/ { PVOID Base; PFN_NUMBER PageFrameIndex; PMMPFN Pfn; MMPTE TempPte; LOGICAL SessionAddress; MM_PFN_LOCK_ASSERT(); Base = MiGetVirtualAddressMappedByPte (PointerPte); SessionAddress = MI_IS_SESSION_PTE (PointerPte); while (PointerPte <= LastPte) { // // The PTE must be carefully checked as drivers may call MmPageEntire // during their DriverEntry yet faults may occur prior to this routine // running which cause pages to already be resident and in the working // set at this point. So checks for validity and wsindex must be // applied. // if (PointerPte->u.Hard.Valid == 1) { PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte); Pfn = MI_PFN_ELEMENT (PageFrameIndex); ASSERT (Pfn->u2.ShareCount == 1); if (Pfn->u1.WsIndex == 0) { // // Set the working set index to zero. This allows page table // pages to be brought back in with the proper WSINDEX. // MI_ZERO_WSINDEX (Pfn); // // Original PTE may need to be set for drivers loaded via // ntldr. // if (Pfn->OriginalPte.u.Long == 0) { Pfn->OriginalPte.u.Long = MM_KERNEL_DEMAND_ZERO_PTE; Pfn->OriginalPte.u.Soft.Protection |= MM_EXECUTE; } MI_SET_MODIFIED (Pfn, 1, 0x11); TempPte = *PointerPte; MI_MAKE_VALID_PTE_TRANSITION (TempPte, Pfn->OriginalPte.u.Soft.Protection); KeFlushSingleTb (Base, TRUE, TRUE, (PHARDWARE_PTE)PointerPte, TempPte.u.Flush); // // Flush the translation buffer and decrement the number of valid // PTEs within the containing page table page. Note that for a // private page, the page table page is still needed because the // page is in transition. // MiDecrementShareCount (PageFrameIndex); MmResidentAvailablePages += 1; MmTotalSystemCodePages += 1; } else { // // This would need to be taken out of the WSLEs so skip it for // now and let the normal paging algorithms remove it if we // run into memory pressure. // } } Base = (PVOID)((PCHAR)Base + PAGE_SIZE); PointerPte += 1; } if (SessionAddress == TRUE) { // // Session space has no ASN - flush the entire TB. // MI_FLUSH_ENTIRE_SESSION_TB (TRUE, TRUE); } return; } MM_SYSTEMSIZE MmQuerySystemSize( VOID ) { // // 12Mb is small // 12-19 is medium // > 19 is large // return MmSystemSize; } NTKERNELAPI BOOLEAN MmIsThisAnNtAsSystem( VOID ) { return (BOOLEAN)MmProductType; } NTKERNELAPI VOID FASTCALL MmSetPageFaultNotifyRoutine( PPAGE_FAULT_NOTIFY_ROUTINE NotifyRoutine ) { MmPageFaultNotifyRoutine = NotifyRoutine; } #ifdef _MI_MESSAGE_SERVER LIST_ENTRY MiMessageInfoListHead; KSPIN_LOCK MiMessageLock; KEVENT MiMessageEvent; ULONG MiMessageCount; VOID MiInitializeMessageQueue ( VOID ) { MiMessageCount = 0; InitializeListHead (&MiMessageInfoListHead); KeInitializeSpinLock (&MiMessageLock); // // Use a synchronization event so the event's signal state is // auto cleared on a successful wait. // KeInitializeEvent (&MiMessageEvent, SynchronizationEvent, FALSE); } LOGICAL MiQueueMessage ( IN PVOID Message ) { KIRQL OldIrql; LOGICAL Enqueued; Enqueued = TRUE; ExAcquireSpinLock (&MiMessageLock, &OldIrql); if (MiMessageCount <= 500) { InsertTailList (&MiMessageInfoListHead, (PLIST_ENTRY)Message); MiMessageCount += 1; } else { Enqueued = FALSE; } ExReleaseSpinLock (&MiMessageLock, OldIrql); if (Enqueued == TRUE) { KeSetEvent (&MiMessageEvent, 0, FALSE); } else { ExFreePool (Message); } return Enqueued; } // // sr: free the pool when done. // PVOID MiRemoveMessage ( VOID ) { PVOID Message; KIRQL OldIrql; NTSTATUS Status; Message = NULL; // // N.B. waiting with a timeout and return so caller can support unload. // Status = KeWaitForSingleObject (&MiMessageEvent, WrVirtualMemory, KernelMode, FALSE, (PLARGE_INTEGER) &MmTwentySeconds); if (Status != STATUS_TIMEOUT) { ExAcquireSpinLock (&MiMessageLock, &OldIrql); if (!IsListEmpty (&MiMessageInfoListHead)) { Message = (PVOID)RemoveHeadList(&MiMessageInfoListHead); MiMessageCount -= 1; if (!IsListEmpty (&MiMessageInfoListHead)) { // // The list still contains entries so undo the event autoreset. // KeSetEvent (&MiMessageEvent, 0, FALSE); } } ExReleaseSpinLock (&MiMessageLock, OldIrql); } return Message; } #endif #define CONSTANT_UNICODE_STRING(s) { sizeof( s ) - sizeof( WCHAR ), sizeof( s ), s } LOGICAL MiInitializeMemoryEvents ( VOID ) { KIRQL OldIrql; NTSTATUS Status; UNICODE_STRING LowMem = CONSTANT_UNICODE_STRING(L"\\KernelObjects\\LowMemoryCondition"); UNICODE_STRING HighMem = CONSTANT_UNICODE_STRING(L"\\KernelObjects\\HighMemoryCondition"); // // The thresholds may be set in the registry, if so, they are interpreted // in megabytes so convert them to pages now. // // If the user modifies the registry to introduce his own values, don't // bother error checking them as they can't hurt the system regardless (bad // values just may result in events not getting signaled or staying // signaled when they shouldn't, but that's not fatal). // if (MmLowMemoryThreshold != 0) { MmLowMemoryThreshold *= ((1024 * 1024) / PAGE_SIZE); } else { // // Scale the threshold so on servers the low threshold is // approximately 32MB per 4GB, capping it at 64MB. // MmLowMemoryThreshold = MmPlentyFreePages; if (MmNumberOfPhysicalPages > 0x40000) { MmLowMemoryThreshold = (32 * 1024 * 1024) / PAGE_SIZE; MmLowMemoryThreshold += ((MmNumberOfPhysicalPages - 0x40000) >> 7); } else if (MmNumberOfPhysicalPages > 0x8000) { MmLowMemoryThreshold += ((MmNumberOfPhysicalPages - 0x8000) >> 5); } if (MmLowMemoryThreshold > (64 * 1024 * 1024) / PAGE_SIZE) { MmLowMemoryThreshold = (64 * 1024 * 1024) / PAGE_SIZE; } } if (MmHighMemoryThreshold != 0) { MmHighMemoryThreshold *= ((1024 * 1024) / PAGE_SIZE); } else { MmHighMemoryThreshold = 3 * MmLowMemoryThreshold; ASSERT (MmHighMemoryThreshold > MmLowMemoryThreshold); } if (MmHighMemoryThreshold < MmLowMemoryThreshold) { MmHighMemoryThreshold = MmLowMemoryThreshold; } Status = MiCreateMemoryEvent (&LowMem, &MiLowMemoryEvent); if (!NT_SUCCESS (Status)) { #if DBG DbgPrint ("MM: Memory event initialization failed %x\n", Status); #endif return FALSE; } Status = MiCreateMemoryEvent (&HighMem, &MiHighMemoryEvent); if (!NT_SUCCESS (Status)) { #if DBG DbgPrint ("MM: Memory event initialization failed %x\n", Status); #endif return FALSE; } // // Initialize the event values. // LOCK_PFN (OldIrql); MiNotifyMemoryEvents (); UNLOCK_PFN (OldIrql); return TRUE; } extern POBJECT_TYPE ExEventObjectType; NTSTATUS MiCreateMemoryEvent ( IN PUNICODE_STRING EventName, OUT PKEVENT *Event ) { PACL Dacl; HANDLE EventHandle; ULONG DaclLength; NTSTATUS Status; OBJECT_ATTRIBUTES ObjectAttributes; SECURITY_DESCRIPTOR SecurityDescriptor; Status = RtlCreateSecurityDescriptor (&SecurityDescriptor, SECURITY_DESCRIPTOR_REVISION); if (!NT_SUCCESS (Status)) { return Status; } DaclLength = sizeof (ACL) + sizeof (ACCESS_ALLOWED_ACE) * 3 + RtlLengthSid (SeLocalSystemSid) + RtlLengthSid (SeAliasAdminsSid) + RtlLengthSid (SeWorldSid); Dacl = ExAllocatePoolWithTag (PagedPool, DaclLength, 'lcaD'); if (Dacl == NULL) { return STATUS_INSUFFICIENT_RESOURCES; } Status = RtlCreateAcl (Dacl, DaclLength, ACL_REVISION); if (!NT_SUCCESS (Status)) { ExFreePool (Dacl); return Status; } Status = RtlAddAccessAllowedAce (Dacl, ACL_REVISION, EVENT_ALL_ACCESS, SeAliasAdminsSid); if (!NT_SUCCESS (Status)) { ExFreePool (Dacl); return Status; } Status = RtlAddAccessAllowedAce (Dacl, ACL_REVISION, EVENT_ALL_ACCESS, SeLocalSystemSid); if (!NT_SUCCESS (Status)) { ExFreePool (Dacl); return Status; } Status = RtlAddAccessAllowedAce (Dacl, ACL_REVISION, SYNCHRONIZE|EVENT_QUERY_STATE|READ_CONTROL, SeWorldSid); if (!NT_SUCCESS (Status)) { ExFreePool (Dacl); return Status; } Status = RtlSetDaclSecurityDescriptor (&SecurityDescriptor, TRUE, Dacl, FALSE); if (!NT_SUCCESS (Status)) { ExFreePool (Dacl); return Status; } InitializeObjectAttributes (&ObjectAttributes, EventName, OBJ_KERNEL_HANDLE | OBJ_PERMANENT, NULL, &SecurityDescriptor); Status = ZwCreateEvent (&EventHandle, EVENT_ALL_ACCESS, &ObjectAttributes, NotificationEvent, FALSE); ExFreePool (Dacl); if (NT_SUCCESS (Status)) { Status = ObReferenceObjectByHandle (EventHandle, EVENT_MODIFY_STATE, ExEventObjectType, KernelMode, (PVOID *)Event, NULL); } ZwClose (EventHandle); return Status; } VOID MiNotifyMemoryEvents ( VOID ) // PFN lock is held. { if (MmAvailablePages <= MmLowMemoryThreshold) { if (KeReadStateEvent (MiHighMemoryEvent) != 0) { KeClearEvent (MiHighMemoryEvent); } if (KeReadStateEvent (MiLowMemoryEvent) == 0) { KeSetEvent (MiLowMemoryEvent, 0, FALSE); } } else if (MmAvailablePages < MmHighMemoryThreshold) { // // Gray zone, make sure both events are cleared. // if (KeReadStateEvent (MiHighMemoryEvent) != 0) { KeClearEvent (MiHighMemoryEvent); } if (KeReadStateEvent (MiLowMemoryEvent) != 0) { KeClearEvent (MiLowMemoryEvent); } } else { if (KeReadStateEvent (MiHighMemoryEvent) == 0) { KeSetEvent (MiHighMemoryEvent, 0, FALSE); } if (KeReadStateEvent (MiLowMemoryEvent) != 0) { KeClearEvent (MiLowMemoryEvent); } } return; } VOID MiInitializeCacheOverrides ( VOID ) { #if defined (_WIN64) ULONG NumberOfBytes; NTSTATUS Status; HAL_PLATFORM_INFORMATION Information; // // Gather platform information from the HAL. // Status = HalQuerySystemInformation (HalPlatformInformation, sizeof (Information), &Information, &NumberOfBytes); if (!NT_SUCCESS (Status)) { return; } // // Apply mapping modifications based on platform information flags. // // It would be better if the platform returned what the new cachetype // should be. // if (Information.PlatformFlags & HAL_PLATFORM_DISABLE_UC_MAIN_MEMORY) { MI_SET_CACHETYPE_TRANSLATION (MmNonCached, 0, MiCached); } #endif return; }