1024 lines
24 KiB
C
1024 lines
24 KiB
C
/*++
|
||
|
||
Copyright (c) 1989 Microsoft Corporation
|
||
|
||
Module Name:
|
||
|
||
compress.c
|
||
|
||
Abstract:
|
||
|
||
This module contains the routines to support allow hardware to
|
||
transparently compress physical memory.
|
||
|
||
Author:
|
||
|
||
Landy Wang (landyw) 21-Oct-2000
|
||
|
||
Revision History:
|
||
|
||
--*/
|
||
|
||
#include "mi.h"
|
||
|
||
#if defined (_MI_COMPRESSION)
|
||
|
||
Enable the #if 0 code in cmdat3.c to allow Ratio specification.
|
||
|
||
//
|
||
// Compression public interface.
|
||
//
|
||
|
||
#define MM_PHYSICAL_MEMORY_PRODUCED_VIA_COMPRESSION 0x1
|
||
|
||
typedef
|
||
NTSTATUS
|
||
(*PMM_SET_COMPRESSION_THRESHOLD) (
|
||
IN ULONGLONG CompressionByteThreshold
|
||
);
|
||
|
||
typedef struct _MM_COMPRESSION_CONTEXT {
|
||
ULONG Version;
|
||
ULONG SizeInBytes;
|
||
ULONGLONG ReservedBytes;
|
||
PMM_SET_COMPRESSION_THRESHOLD SetCompressionThreshold;
|
||
} MM_COMPRESSION_CONTEXT, *PMM_COMPRESSION_CONTEXT;
|
||
|
||
#define MM_COMPRESSION_VERSION_INITIAL 1
|
||
#define MM_COMPRESSION_VERSION_CURRENT 1
|
||
|
||
NTSTATUS
|
||
MmRegisterCompressionDevice (
|
||
IN PMM_COMPRESSION_CONTEXT Context
|
||
);
|
||
|
||
NTSTATUS
|
||
MmDeregisterCompressionDevice (
|
||
IN PMM_COMPRESSION_CONTEXT Context
|
||
);
|
||
|
||
//
|
||
// This defaults to 75% but can be overridden in the registry. At this
|
||
// percentage of *real* physical memory in use, an interrupt is generated so
|
||
// that memory management can zero pages to make more memory available.
|
||
//
|
||
|
||
#define MI_DEFAULT_COMPRESSION_THRESHOLD 75
|
||
|
||
ULONG MmCompressionThresholdRatio;
|
||
|
||
PFN_NUMBER MiNumberOfCompressionPages;
|
||
|
||
PMM_SET_COMPRESSION_THRESHOLD MiSetCompressionThreshold;
|
||
|
||
#if DBG
|
||
KIRQL MiCompressionIrql;
|
||
#endif
|
||
|
||
//
|
||
// Note there is also code in dynmem.c that is dependent on this #define.
|
||
//
|
||
|
||
#if defined (_MI_COMPRESSION_SUPPORTED_)
|
||
|
||
typedef struct _MI_COMPRESSION_INFO {
|
||
ULONG IsrPageProcessed;
|
||
ULONG DpcPageProcessed;
|
||
ULONG IsrForcedDpc;
|
||
ULONG IsrFailedDpc;
|
||
|
||
ULONG IsrRan;
|
||
ULONG DpcRan;
|
||
ULONG DpcsFired;
|
||
ULONG IsrSkippedZeroedPage;
|
||
|
||
ULONG DpcSkippedZeroedPage;
|
||
ULONG CtxswapForcedDpcInsert;
|
||
ULONG CtxswapFailedDpcInsert;
|
||
ULONG PfnForcedDpcInsert;
|
||
|
||
ULONG PfnFailedDpcInsert;
|
||
|
||
} MI_COMPRESSION_INFO, *PMI_COMPRESSION_INFO;
|
||
|
||
MI_COMPRESSION_INFO MiCompressionInfo; // LWFIX - temp remove.
|
||
|
||
PFN_NUMBER MiCompressionOverHeadInPages;
|
||
|
||
PKDPC MiCompressionDpcArray;
|
||
CCHAR MiCompressionProcessors;
|
||
|
||
VOID
|
||
MiCompressionDispatch (
|
||
IN PKDPC Dpc,
|
||
IN PVOID DeferredContext,
|
||
IN PVOID SystemArgument1,
|
||
IN PVOID SystemArgument2
|
||
);
|
||
|
||
PVOID
|
||
MiMapCompressionInHyperSpace (
|
||
IN PFN_NUMBER PageFrameIndex
|
||
);
|
||
|
||
VOID
|
||
MiUnmapCompressionInHyperSpace (
|
||
VOID
|
||
);
|
||
|
||
SIZE_T
|
||
MiMakeCompressibleMemoryAtDispatch (
|
||
IN SIZE_T NumberOfBytes OPTIONAL
|
||
);
|
||
|
||
|
||
NTSTATUS
|
||
MmRegisterCompressionDevice (
|
||
IN PMM_COMPRESSION_CONTEXT Context
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This routine notifies memory management that compression hardware exists
|
||
in the system. Memory management responds by initializing compression
|
||
support here.
|
||
|
||
Arguments:
|
||
|
||
Context - Supplies the compression context pointer.
|
||
|
||
Return Value:
|
||
|
||
NTSTATUS.
|
||
|
||
Environment:
|
||
|
||
Kernel mode, PASSIVE_LEVEL.
|
||
|
||
--*/
|
||
|
||
{
|
||
KIRQL OldIrql;
|
||
PFN_NUMBER OverHeadInPages;
|
||
CCHAR Processor;
|
||
CCHAR NumberProcessors;
|
||
PKDPC CompressionDpcArray;
|
||
|
||
ASSERT (KeGetCurrentIrql () == PASSIVE_LEVEL);
|
||
|
||
if (Context->Version != MM_COMPRESSION_VERSION_CURRENT) {
|
||
return STATUS_INVALID_PARAMETER_1;
|
||
}
|
||
|
||
if (Context->SizeInBytes < sizeof (MM_COMPRESSION_CONTEXT)) {
|
||
return STATUS_INVALID_PARAMETER_1;
|
||
}
|
||
|
||
//
|
||
// If the subsequent hot-add cannot succeed then fail this API now.
|
||
//
|
||
|
||
if (MmDynamicPfn == 0) {
|
||
return STATUS_NOT_SUPPORTED;
|
||
}
|
||
|
||
//
|
||
// Hardware that can't generate a configurable interrupt is not supported.
|
||
//
|
||
|
||
if (Context->SetCompressionThreshold == NULL) {
|
||
return STATUS_INVALID_PARAMETER_1;
|
||
}
|
||
|
||
//
|
||
// ReservedBytes indicates the number of reserved bytes required by the
|
||
// underlying hardware. For example, some hardware might have:
|
||
//
|
||
// 1. translation tables which are 1/64 of the fictional RAM total.
|
||
//
|
||
// 2. the first MB of memory is never compressed.
|
||
//
|
||
// 3. an L3 which is never compressed.
|
||
//
|
||
// etc.
|
||
//
|
||
// ReservedBytes would be the sum of all of these types of ranges.
|
||
//
|
||
|
||
OverHeadInPages = (PFN_COUNT)(Context->ReservedBytes / PAGE_SIZE);
|
||
|
||
if (MmResidentAvailablePages < (SPFN_NUMBER) OverHeadInPages) {
|
||
return STATUS_INSUFFICIENT_RESOURCES;
|
||
}
|
||
|
||
if (MmAvailablePages < OverHeadInPages) {
|
||
MmEmptyAllWorkingSets ();
|
||
if (MmAvailablePages < OverHeadInPages) {
|
||
return STATUS_INSUFFICIENT_RESOURCES;
|
||
}
|
||
}
|
||
|
||
//
|
||
// Create a DPC for every processor in the system as servicing the
|
||
// compression interrupt is critical.
|
||
//
|
||
|
||
NumberProcessors = KeNumberProcessors;
|
||
|
||
CompressionDpcArray = ExAllocatePoolWithTag (NonPagedPool,
|
||
NumberProcessors * sizeof (KDPC),
|
||
'pDmM');
|
||
|
||
if (CompressionDpcArray == NULL) {
|
||
return STATUS_INSUFFICIENT_RESOURCES;
|
||
}
|
||
|
||
for (Processor = 0; Processor < NumberProcessors; Processor += 1) {
|
||
|
||
KeInitializeDpc (CompressionDpcArray + Processor, MiCompressionDispatch, NULL);
|
||
|
||
//
|
||
// Set importance so this DPC always gets queued at the head.
|
||
//
|
||
|
||
KeSetImportanceDpc (CompressionDpcArray + Processor, HighImportance);
|
||
|
||
KeSetTargetProcessorDpc (CompressionDpcArray + Processor, Processor);
|
||
}
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
if (MmCompressionThresholdRatio == 0) {
|
||
MmCompressionThresholdRatio = MI_DEFAULT_COMPRESSION_THRESHOLD;
|
||
}
|
||
else if (MmCompressionThresholdRatio > 100) {
|
||
MmCompressionThresholdRatio = 100;
|
||
}
|
||
|
||
if ((MmResidentAvailablePages < (SPFN_NUMBER) OverHeadInPages) ||
|
||
(MmAvailablePages < OverHeadInPages)) {
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
ExFreePool (CompressionDpcArray);
|
||
return STATUS_INSUFFICIENT_RESOURCES;
|
||
}
|
||
|
||
MmResidentAvailablePages -= OverHeadInPages;
|
||
MmAvailablePages -= (PFN_COUNT) OverHeadInPages;
|
||
|
||
//
|
||
// Snap our own copy to prevent busted drivers from causing overcommits
|
||
// if they deregister improperly.
|
||
//
|
||
|
||
MiCompressionOverHeadInPages += OverHeadInPages;
|
||
|
||
ASSERT (MiNumberOfCompressionPages == 0);
|
||
|
||
ASSERT (MiSetCompressionThreshold == NULL);
|
||
MiSetCompressionThreshold = Context->SetCompressionThreshold;
|
||
|
||
if (MiCompressionDpcArray == NULL) {
|
||
MiCompressionDpcArray = CompressionDpcArray;
|
||
CompressionDpcArray = NULL;
|
||
MiCompressionProcessors = NumberProcessors;
|
||
}
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
|
||
if (CompressionDpcArray != NULL) {
|
||
ExFreePool (CompressionDpcArray);
|
||
}
|
||
|
||
return STATUS_SUCCESS;
|
||
}
|
||
|
||
|
||
NTSTATUS
|
||
MiArmCompressionInterrupt (
|
||
VOID
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This routine arms the hardware-generated compression interrupt.
|
||
|
||
Arguments:
|
||
|
||
None.
|
||
|
||
Return Value:
|
||
|
||
NTSTATUS.
|
||
|
||
Environment:
|
||
|
||
Kernel mode, PFN lock held.
|
||
|
||
--*/
|
||
|
||
{
|
||
NTSTATUS Status;
|
||
PFN_NUMBER RealPages;
|
||
ULONGLONG ByteThreshold;
|
||
|
||
MM_PFN_LOCK_ASSERT();
|
||
|
||
if (MiSetCompressionThreshold == NULL) {
|
||
return STATUS_SUCCESS;
|
||
}
|
||
|
||
RealPages = MmNumberOfPhysicalPages - MiNumberOfCompressionPages - MiCompressionOverHeadInPages;
|
||
|
||
ByteThreshold = (RealPages * MmCompressionThresholdRatio) / 100;
|
||
ByteThreshold *= PAGE_SIZE;
|
||
|
||
//
|
||
// Note this callout is made with the PFN lock held !
|
||
//
|
||
|
||
Status = (*MiSetCompressionThreshold) (ByteThreshold);
|
||
|
||
if (!NT_SUCCESS (Status)) {
|
||
|
||
//
|
||
// If the hardware fails, all is lost.
|
||
//
|
||
|
||
KeBugCheckEx (MEMORY_MANAGEMENT,
|
||
0x61941,
|
||
MmNumberOfPhysicalPages,
|
||
RealPages,
|
||
MmCompressionThresholdRatio);
|
||
}
|
||
|
||
return Status;
|
||
}
|
||
|
||
|
||
NTSTATUS
|
||
MmDeregisterCompressionDevice (
|
||
IN PMM_COMPRESSION_CONTEXT Context
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This routine notifies memory management that compression hardware is
|
||
being removed. Note the compression driver must have already SUCCESSFULLY
|
||
called MmRemovePhysicalMemoryEx.
|
||
|
||
Arguments:
|
||
|
||
Context - Supplies the compression context pointer.
|
||
|
||
Return Value:
|
||
|
||
STATUS_SUCCESS if compression support is initialized properly.
|
||
|
||
Environment:
|
||
|
||
Kernel mode, PASSIVE_LEVEL.
|
||
|
||
--*/
|
||
|
||
{
|
||
KIRQL OldIrql;
|
||
PFN_COUNT OverHeadInPages;
|
||
|
||
ASSERT (KeGetCurrentIrql () == PASSIVE_LEVEL);
|
||
|
||
OverHeadInPages = (PFN_COUNT)(Context->ReservedBytes / PAGE_SIZE);
|
||
|
||
LOCK_PFN (OldIrql);
|
||
|
||
if (OverHeadInPages > MiCompressionOverHeadInPages) {
|
||
UNLOCK_PFN (OldIrql);
|
||
return STATUS_INVALID_PARAMETER;
|
||
}
|
||
|
||
MmResidentAvailablePages += OverHeadInPages;
|
||
MmAvailablePages += OverHeadInPages;
|
||
|
||
ASSERT (MiCompressionOverHeadInPages == OverHeadInPages);
|
||
|
||
MiCompressionOverHeadInPages -= OverHeadInPages;
|
||
|
||
MiSetCompressionThreshold = NULL;
|
||
|
||
UNLOCK_PFN (OldIrql);
|
||
|
||
return STATUS_SUCCESS;
|
||
}
|
||
|
||
VOID
|
||
MiCompressionDispatch (
|
||
IN PKDPC Dpc,
|
||
IN PVOID DeferredContext,
|
||
IN PVOID SystemArgument1,
|
||
IN PVOID SystemArgument2
|
||
)
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
Called to make memory compressible if the PFN lock could not be
|
||
acquired during the original device interrupt.
|
||
|
||
Arguments:
|
||
|
||
Dpc - Supplies a pointer to a control object of type DPC.
|
||
|
||
SystemArgument1 - Supplies the number of bytes to make compressible.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Environment:
|
||
|
||
Kernel mode. DISPATCH_LEVEL.
|
||
|
||
--*/
|
||
{
|
||
SIZE_T NumberOfBytes;
|
||
|
||
UNREFERENCED_PARAMETER (Dpc);
|
||
UNREFERENCED_PARAMETER (DeferredContext);
|
||
UNREFERENCED_PARAMETER (SystemArgument2);
|
||
|
||
NumberOfBytes = (SIZE_T) SystemArgument1;
|
||
|
||
MiCompressionInfo.DpcsFired += 1;
|
||
|
||
MiMakeCompressibleMemoryAtDispatch (NumberOfBytes);
|
||
}
|
||
|
||
SIZE_T
|
||
MmMakeCompressibleMemory (
|
||
IN SIZE_T NumberOfBytes OPTIONAL
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This routine attempts to move pages from transition to zero so that
|
||
hardware compression can reclaim the physical memory.
|
||
|
||
Arguments:
|
||
|
||
NumberOfBytes - Supplies the number of bytes to make compressible.
|
||
Zero indicates as much as possible.
|
||
|
||
Return Value:
|
||
|
||
Returns the number of bytes made compressible.
|
||
|
||
Environment:
|
||
|
||
Kernel mode. Any IRQL as this is called from device interrupt service
|
||
routines.
|
||
|
||
--*/
|
||
|
||
{
|
||
KIRQL OldIrql;
|
||
BOOLEAN Queued;
|
||
#if !defined(NT_UP)
|
||
PFN_NUMBER PageFrameIndex;
|
||
MMLISTS MemoryList;
|
||
PMMPFNLIST ListHead;
|
||
PMMPFN Pfn1;
|
||
CCHAR Processor;
|
||
PFN_NUMBER Total;
|
||
PVOID ZeroBase;
|
||
PKPRCB Prcb;
|
||
PFN_NUMBER RequestedPages;
|
||
PFN_NUMBER ActualPages;
|
||
PKSPIN_LOCK_QUEUE LockQueuePfn;
|
||
PKSPIN_LOCK_QUEUE LockQueueContextSwap;
|
||
#endif
|
||
|
||
//
|
||
// LWFIX: interlocked add in the request size above so overlapping
|
||
// requests can be processed.
|
||
//
|
||
|
||
OldIrql = KeGetCurrentIrql();
|
||
|
||
if (OldIrql <= DISPATCH_LEVEL) {
|
||
return MiMakeCompressibleMemoryAtDispatch (NumberOfBytes);
|
||
}
|
||
|
||
#if defined(NT_UP)
|
||
|
||
//
|
||
// In uniprocessor configurations, there is no indication as to whether
|
||
// various locks of interest (context swap & PFN) are owned because the
|
||
// uniprocessor kernel macros these into merely IRQL raises. Therefore
|
||
// this routine must be conservative when called above DISPATCH_LEVEL and
|
||
// assume the lock is owned and just always queue a DPC in these cases.
|
||
//
|
||
|
||
Queued = KeInsertQueueDpc (MiCompressionDpcArray,
|
||
(PVOID) NumberOfBytes,
|
||
NULL);
|
||
if (Queued == TRUE) {
|
||
MiCompressionInfo.CtxswapForcedDpcInsert += 1;
|
||
}
|
||
else {
|
||
MiCompressionInfo.CtxswapFailedDpcInsert += 1;
|
||
}
|
||
|
||
return 0;
|
||
|
||
#else
|
||
|
||
#if DBG
|
||
//
|
||
// Make sure this interrupt always comes in at the same device IRQL.
|
||
//
|
||
|
||
ASSERT ((MiCompressionIrql == 0) || (OldIrql == MiCompressionIrql));
|
||
MiCompressionIrql = OldIrql;
|
||
#endif
|
||
|
||
Prcb = KeGetCurrentPrcb();
|
||
|
||
LockQueueContextSwap = &Prcb->LockQueue[LockQueueContextSwapLock];
|
||
|
||
//
|
||
// The context swap lock is needed for TB flushes. The interrupted thread
|
||
// may already own this in the midst of doing a TB flush without the PFN
|
||
// lock being held. Since there is no safe way to tell if the current
|
||
// processor owns the context swap lock, just do a try-acquire followed
|
||
// by an immediate release to decide if it is safe to proceed.
|
||
//
|
||
|
||
if (KeTryToAcquireQueuedSpinLockAtRaisedIrql (LockQueueContextSwap) == FALSE) {
|
||
|
||
//
|
||
// Unable to acquire the spinlock, queue a DPC to pick it up instead.
|
||
//
|
||
|
||
for (Processor = 0; Processor < MiCompressionProcessors; Processor += 1) {
|
||
|
||
Queued = KeInsertQueueDpc (MiCompressionDpcArray + Processor,
|
||
(PVOID) NumberOfBytes,
|
||
NULL);
|
||
if (Queued == TRUE) {
|
||
MiCompressionInfo.CtxswapForcedDpcInsert += 1;
|
||
}
|
||
else {
|
||
MiCompressionInfo.CtxswapFailedDpcInsert += 1;
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
KeReleaseQueuedSpinLockFromDpcLevel (LockQueueContextSwap);
|
||
|
||
RequestedPages = NumberOfBytes >> PAGE_SHIFT;
|
||
ActualPages = 0;
|
||
|
||
MemoryList = FreePageList;
|
||
|
||
ListHead = MmPageLocationList[MemoryList];
|
||
|
||
LockQueuePfn = &Prcb->LockQueue[LockQueuePfnLock];
|
||
|
||
if (KeTryToAcquireQueuedSpinLockAtRaisedIrql (LockQueuePfn) == FALSE) {
|
||
|
||
//
|
||
// Unable to acquire the spinlock, queue a DPC to pick it up instead.
|
||
//
|
||
|
||
for (Processor = 0; Processor < MiCompressionProcessors; Processor += 1) {
|
||
|
||
Queued = KeInsertQueueDpc (MiCompressionDpcArray + Processor,
|
||
(PVOID) NumberOfBytes,
|
||
NULL);
|
||
if (Queued == TRUE) {
|
||
MiCompressionInfo.PfnForcedDpcInsert += 1;
|
||
}
|
||
else {
|
||
MiCompressionInfo.PfnFailedDpcInsert += 1;
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
MiCompressionInfo.IsrRan += 1;
|
||
|
||
//
|
||
// Run the free and transition list and zero the pages.
|
||
//
|
||
|
||
while (MemoryList <= StandbyPageList) {
|
||
|
||
Total = ListHead->Total;
|
||
|
||
PageFrameIndex = ListHead->Flink;
|
||
|
||
while (Total != 0) {
|
||
|
||
//
|
||
// Transition pages may need restoration which requires a
|
||
// hyperspace mapping plus control area deletion actions all of
|
||
// which occur at DISPATCH_LEVEL. So if we're at device IRQL,
|
||
// only do the minimum and queue the rest.
|
||
//
|
||
|
||
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
||
|
||
if ((Pfn1->u3.e1.InPageError == 1) &&
|
||
(Pfn1->u3.e1.ReadInProgress == 1)) {
|
||
|
||
//
|
||
// This page is already zeroed so skip it.
|
||
//
|
||
|
||
MiCompressionInfo.IsrSkippedZeroedPage += 1;
|
||
}
|
||
else {
|
||
|
||
//
|
||
// Zero the page directly now instead of waiting for the low
|
||
// priority zeropage thread to get a slice. Note that the
|
||
// slower mapping and zeroing routines are used here because
|
||
// the faster ones are for the zeropage thread only.
|
||
// Maybe we should change this someday.
|
||
//
|
||
|
||
ZeroBase = MiMapCompressionInHyperSpace (PageFrameIndex);
|
||
|
||
RtlZeroMemory (ZeroBase, PAGE_SIZE);
|
||
|
||
MiUnmapCompressionInHyperSpace ();
|
||
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
|
||
//
|
||
// Overload ReadInProgress to signify that collided faults that
|
||
// occur before the PTE is completely restored will know to
|
||
// delay and retry until the page (and PTE) are updated.
|
||
//
|
||
|
||
Pfn1->u3.e1.InPageError = 1;
|
||
ASSERT (Pfn1->u3.e1.ReadInProgress == 0);
|
||
Pfn1->u3.e1.ReadInProgress = 1;
|
||
|
||
ActualPages += 1;
|
||
|
||
if (ActualPages == RequestedPages) {
|
||
MemoryList = StandbyPageList;
|
||
ListHead = MmPageLocationList[MemoryList];
|
||
break;
|
||
}
|
||
}
|
||
|
||
Total -= 1;
|
||
PageFrameIndex = Pfn1->u1.Flink;
|
||
}
|
||
MemoryList += 1;
|
||
ListHead += 1;
|
||
}
|
||
|
||
if (ActualPages != 0) {
|
||
|
||
//
|
||
// Rearm the interrupt as pages have now been zeroed.
|
||
//
|
||
|
||
MiArmCompressionInterrupt ();
|
||
}
|
||
|
||
KeReleaseQueuedSpinLockFromDpcLevel (LockQueuePfn);
|
||
|
||
if (ActualPages != 0) {
|
||
|
||
//
|
||
// Pages were zeroed - queue a DPC to the current processor to
|
||
// move them to the zero list. Note this is not critical path so
|
||
// don't bother sending a DPC to every processor for this case.
|
||
//
|
||
|
||
MiCompressionInfo.IsrPageProcessed += (ULONG)ActualPages;
|
||
|
||
Processor = (CCHAR) KeGetCurrentProcessorNumber ();
|
||
|
||
//
|
||
// Ensure a hot-added processor scenario just works.
|
||
//
|
||
|
||
if (Processor >= MiCompressionProcessors) {
|
||
Processor = MiCompressionProcessors;
|
||
}
|
||
|
||
Queued = KeInsertQueueDpc (MiCompressionDpcArray + Processor,
|
||
(PVOID) NumberOfBytes,
|
||
NULL);
|
||
if (Queued == TRUE) {
|
||
MiCompressionInfo.IsrForcedDpc += 1;
|
||
}
|
||
else {
|
||
MiCompressionInfo.IsrFailedDpc += 1;
|
||
}
|
||
}
|
||
|
||
return (ActualPages << PAGE_SHIFT);
|
||
#endif
|
||
}
|
||
|
||
SIZE_T
|
||
MiMakeCompressibleMemoryAtDispatch (
|
||
IN SIZE_T NumberOfBytes OPTIONAL
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This routine attempts to move pages from transition to zero so that
|
||
hardware compression can reclaim the physical memory.
|
||
|
||
Arguments:
|
||
|
||
NumberOfBytes - Supplies the number of bytes to make compressible.
|
||
Zero indicates as much as possible.
|
||
|
||
Return Value:
|
||
|
||
Returns the number of bytes made compressible.
|
||
|
||
Environment:
|
||
|
||
Kernel mode. DISPATCH_LEVEL.
|
||
|
||
--*/
|
||
|
||
{
|
||
KIRQL OldIrql;
|
||
PFN_NUMBER PageFrameIndex;
|
||
PFN_NUMBER PageFrameIndex2;
|
||
PVOID ZeroBase;
|
||
PMMPFN Pfn1;
|
||
MMLISTS MemoryList;
|
||
PMMPFNLIST ListHead;
|
||
PFN_NUMBER RequestedPages;
|
||
PFN_NUMBER ActualPages;
|
||
LOGICAL NeedToZero;
|
||
|
||
ASSERT (KeGetCurrentIrql () == DISPATCH_LEVEL);
|
||
|
||
RequestedPages = NumberOfBytes >> PAGE_SHIFT;
|
||
ActualPages = 0;
|
||
|
||
MemoryList = FreePageList;
|
||
ListHead = MmPageLocationList[MemoryList];
|
||
|
||
MiCompressionInfo.DpcRan += 1;
|
||
|
||
LOCK_PFN2 (OldIrql);
|
||
|
||
//
|
||
// Run the free and transition list and zero the pages.
|
||
//
|
||
|
||
while (MemoryList <= StandbyPageList) {
|
||
|
||
while (ListHead->Total != 0) {
|
||
|
||
//
|
||
// Before removing the page from the head of the list (which will
|
||
// zero the flag bits), snap whether it's been zeroed by our ISR
|
||
// or whether we need to zero it here.
|
||
//
|
||
|
||
PageFrameIndex = ListHead->Flink;
|
||
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
|
||
|
||
NeedToZero = TRUE;
|
||
if ((Pfn1->u3.e1.InPageError == 1) && (Pfn1->u3.e1.ReadInProgress == 1)) {
|
||
MiCompressionInfo.DpcSkippedZeroedPage += 1;
|
||
NeedToZero = FALSE;
|
||
}
|
||
|
||
//
|
||
// Transition pages may need restoration which requires a
|
||
// hyperspace mapping plus control area deletion actions all of
|
||
// which occur at DISPATCH_LEVEL. Since we're at DISPATCH_LEVEL
|
||
// now, go ahead and do it.
|
||
//
|
||
|
||
PageFrameIndex2 = MiRemovePageFromList (ListHead);
|
||
ASSERT (PageFrameIndex == PageFrameIndex2);
|
||
|
||
//
|
||
// Zero the page directly now instead of waiting for the low
|
||
// priority zeropage thread to get a slice. Note that the
|
||
// slower mapping and zeroing routines are used here because
|
||
// the faster ones are for the zeropage thread only.
|
||
// Maybe we should change this someday.
|
||
//
|
||
|
||
if (NeedToZero == TRUE) {
|
||
ZeroBase = MiMapCompressionInHyperSpace (PageFrameIndex);
|
||
|
||
RtlZeroMemory (ZeroBase, PAGE_SIZE);
|
||
|
||
MiUnmapCompressionInHyperSpace ();
|
||
}
|
||
|
||
ASSERT (Pfn1->u2.ShareCount == 0);
|
||
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
|
||
|
||
MiInsertPageInList (&MmZeroedPageListHead, PageFrameIndex);
|
||
|
||
//
|
||
// We have changed (zeroed) the contents of this page.
|
||
// If memory mirroring is in progress, the bitmap must be updated.
|
||
//
|
||
|
||
if (MiMirroringActive == TRUE) {
|
||
RtlSetBit (MiMirrorBitMap2, (ULONG)PageFrameIndex);
|
||
}
|
||
|
||
MiCompressionInfo.DpcPageProcessed += 1;
|
||
ActualPages += 1;
|
||
|
||
if (ActualPages == RequestedPages) {
|
||
MemoryList = StandbyPageList;
|
||
ListHead = MmPageLocationList[MemoryList];
|
||
break;
|
||
}
|
||
}
|
||
MemoryList += 1;
|
||
ListHead += 1;
|
||
}
|
||
|
||
//
|
||
// Rearm the interrupt as pages have now been zeroed.
|
||
//
|
||
|
||
MiArmCompressionInterrupt ();
|
||
|
||
UNLOCK_PFN2 (OldIrql);
|
||
|
||
return (ActualPages << PAGE_SHIFT);
|
||
}
|
||
|
||
PVOID
|
||
MiMapCompressionInHyperSpace (
|
||
IN PFN_NUMBER PageFrameIndex
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure maps the specified physical page into the
|
||
PTE within hyper space reserved explicitly for compression page
|
||
mapping.
|
||
|
||
The PTE is guaranteed to always be available since the PFN lock is held.
|
||
|
||
Arguments:
|
||
|
||
PageFrameIndex - Supplies the physical page number to map.
|
||
|
||
Return Value:
|
||
|
||
Returns the virtual address where the specified physical page was
|
||
mapped.
|
||
|
||
Environment:
|
||
|
||
Kernel mode, PFN lock held, any IRQL.
|
||
|
||
--*/
|
||
|
||
{
|
||
MMPTE TempPte;
|
||
PMMPTE PointerPte;
|
||
PVOID FlushVaPointer;
|
||
|
||
ASSERT (PageFrameIndex != 0);
|
||
|
||
TempPte = ValidPtePte;
|
||
TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
|
||
|
||
FlushVaPointer = (PVOID) (ULONG_PTR) COMPRESSION_MAPPING_PTE;
|
||
|
||
//
|
||
// Ensure both modified and accessed bits are set so the hardware doesn't
|
||
// ever write this PTE.
|
||
//
|
||
|
||
ASSERT (TempPte.u.Hard.Dirty == 1);
|
||
ASSERT (TempPte.u.Hard.Accessed == 1);
|
||
|
||
PointerPte = MiGetPteAddress (COMPRESSION_MAPPING_PTE);
|
||
ASSERT (PointerPte->u.Long == 0);
|
||
|
||
//
|
||
// Only flush the TB on the current processor as no context switch can
|
||
// occur while using this mapping.
|
||
//
|
||
|
||
KeFlushSingleTb (FlushVaPointer,
|
||
TRUE,
|
||
FALSE,
|
||
(PHARDWARE_PTE) PointerPte,
|
||
TempPte.u.Flush);
|
||
|
||
return (PVOID) MiGetVirtualAddressMappedByPte (PointerPte);
|
||
}
|
||
|
||
__forceinline
|
||
VOID
|
||
MiUnmapCompressionInHyperSpace (
|
||
VOID
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This procedure unmaps the PTE reserved for mapping the compression page.
|
||
|
||
Arguments:
|
||
|
||
None.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
Environment:
|
||
|
||
Kernel mode, PFN lock held, any IRQL.
|
||
|
||
--*/
|
||
|
||
{
|
||
PMMPTE PointerPte;
|
||
|
||
PointerPte = MiGetPteAddress (COMPRESSION_MAPPING_PTE);
|
||
|
||
//
|
||
// Capture the number of waiters.
|
||
//
|
||
|
||
ASSERT (PointerPte->u.Long != 0);
|
||
|
||
PointerPte->u.Long = 0;
|
||
|
||
return;
|
||
}
|
||
#else
|
||
NTSTATUS
|
||
MmRegisterCompressionDevice (
|
||
IN PMM_COMPRESSION_CONTEXT Context
|
||
)
|
||
{
|
||
UNREFERENCED_PARAMETER (Context);
|
||
|
||
return STATUS_NOT_SUPPORTED;
|
||
}
|
||
|
||
NTSTATUS
|
||
MmDeregisterCompressionDevice (
|
||
IN PMM_COMPRESSION_CONTEXT Context
|
||
)
|
||
{
|
||
UNREFERENCED_PARAMETER (Context);
|
||
|
||
return STATUS_NOT_SUPPORTED;
|
||
}
|
||
SIZE_T
|
||
MmMakeCompressibleMemory (
|
||
IN SIZE_T NumberOfBytes OPTIONAL
|
||
)
|
||
{
|
||
UNREFERENCED_PARAMETER (NumberOfBytes);
|
||
|
||
return 0;
|
||
}
|
||
NTSTATUS
|
||
MiArmCompressionInterrupt (
|
||
VOID
|
||
)
|
||
{
|
||
return STATUS_NOT_SUPPORTED;
|
||
}
|
||
#endif
|
||
|
||
#endif
|