1302 lines
40 KiB
C
1302 lines
40 KiB
C
|
/*++
|
|||
|
|
|
|||
|
|
Copyright (c) 1992 Microsoft Corporation
|
|||
|
|
|
|||
|
|
Module Name:
|
|||
|
|
|
|||
|
|
heappriv.h
|
|||
|
|
|
|||
|
|
Abstract:
|
|||
|
|
|
|||
|
|
Private include file used by heap allocator (heap.c, heapdll.c and
|
|||
|
|
heapdbg.c)
|
|||
|
|
|
|||
|
|
Author:
|
|||
|
|
|
|||
|
|
Steve Wood (stevewo) 25-Oct-1994
|
|||
|
|
|
|||
|
|
Revision History:
|
|||
|
|
|
|||
|
|
--*/
|
|||
|
|
|
|||
|
|
#ifndef _RTL_HEAP_PRIVATE_
|
|||
|
|
#define _RTL_HEAP_PRIVATE_
|
|||
|
|
|
|||
|
|
#include "heappage.h"
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// In private builds (PRERELEASE = 1) we allow using the new low fragmentation heap
|
|||
|
|
// for processes that set the DisableLookaside registry key. The main purpose is to
|
|||
|
|
// allow testing the new heap API.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#ifndef PRERELEASE
|
|||
|
|
|
|||
|
|
#define DISABLE_REGISTRY_TEST_HOOKS
|
|||
|
|
|
|||
|
|
#endif
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Disable FPO optimization so even retail builds get somewhat reasonable
|
|||
|
|
// stack backtraces
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#if i386
|
|||
|
|
// #pragma optimize("y",off)
|
|||
|
|
#endif
|
|||
|
|
|
|||
|
|
#if DBG
|
|||
|
|
#define HEAPASSERT(exp) if (!(exp)) RtlAssert( #exp, __FILE__, __LINE__, NULL )
|
|||
|
|
#else
|
|||
|
|
#define HEAPASSERT(exp)
|
|||
|
|
#endif
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Define Minimum lookaside list depth.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define MINIMUM_LOOKASIDE_DEPTH 4
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This variable contains the fill pattern used for heap tail checking
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
extern const UCHAR CheckHeapFillPattern[ CHECK_HEAP_TAIL_SIZE ];
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Here are the locking routines for the heap (kernel and user)
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#ifdef NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Kernel mode heap uses the kernel resource package for locking
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlInitializeLockRoutine(L) ExInitializeResourceLite((PERESOURCE)(L))
|
|||
|
|
#define RtlAcquireLockRoutine(L) ExAcquireResourceExclusiveLite((PERESOURCE)(L),TRUE)
|
|||
|
|
#define RtlReleaseLockRoutine(L) ExReleaseResourceLite((PERESOURCE)(L))
|
|||
|
|
#define RtlDeleteLockRoutine(L) ExDeleteResourceLite((PERESOURCE)(L))
|
|||
|
|
#define RtlOkayToLockRoutine(L) ExOkayToLockRoutineLite((PERESOURCE)(L))
|
|||
|
|
|
|||
|
|
#else // #ifdef NTOS_KERNEL_ROUTINE
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// User mode heap uses the critical section package for locking
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#ifndef PREALLOCATE_EVENT_MASK
|
|||
|
|
|
|||
|
|
#define PREALLOCATE_EVENT_MASK 0x80000000 // Defined only in dll\resource.c
|
|||
|
|
|
|||
|
|
#endif // PREALLOCATE_EVENT_MASK
|
|||
|
|
|
|||
|
|
#define RtlInitializeLockRoutine(L) RtlInitializeCriticalSectionAndSpinCount((PRTL_CRITICAL_SECTION)(L),(PREALLOCATE_EVENT_MASK | 4000))
|
|||
|
|
#define RtlAcquireLockRoutine(L) RtlEnterCriticalSection((PRTL_CRITICAL_SECTION)(L))
|
|||
|
|
#define RtlReleaseLockRoutine(L) RtlLeaveCriticalSection((PRTL_CRITICAL_SECTION)(L))
|
|||
|
|
#define RtlDeleteLockRoutine(L) RtlDeleteCriticalSection((PRTL_CRITICAL_SECTION)(L))
|
|||
|
|
#define RtlOkayToLockRoutine(L) NtdllOkayToLockRoutine((PVOID)(L))
|
|||
|
|
|
|||
|
|
#endif // #ifdef NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Here are some debugging macros for the heap
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#ifdef NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
#define HEAP_DEBUG_FLAGS 0
|
|||
|
|
#define DEBUG_HEAP(F) FALSE
|
|||
|
|
#define SET_LAST_STATUS(S) NOTHING;
|
|||
|
|
|
|||
|
|
#else // #ifdef NTOS_KERNEL_ROUTINE
|
|||
|
|
|
|||
|
|
#define HEAP_DEBUG_FLAGS (HEAP_VALIDATE_PARAMETERS_ENABLED | \
|
|||
|
|
HEAP_VALIDATE_ALL_ENABLED | \
|
|||
|
|
HEAP_CAPTURE_STACK_BACKTRACES | \
|
|||
|
|
HEAP_CREATE_ENABLE_TRACING | \
|
|||
|
|
HEAP_FLAG_PAGE_ALLOCS)
|
|||
|
|
#define DEBUG_HEAP(F) ((F & HEAP_DEBUG_FLAGS) && !(F & HEAP_SKIP_VALIDATION_CHECKS))
|
|||
|
|
#define SET_LAST_STATUS(S) {NtCurrentTeb()->LastErrorValue = RtlNtStatusToDosError( NtCurrentTeb()->LastStatusValue = (ULONG)(S) );}
|
|||
|
|
|
|||
|
|
#endif // #ifdef NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Here are the macros used for debug printing and breakpoints
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#ifdef NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
#define HeapDebugPrint( _x_ ) {DbgPrint _x_;}
|
|||
|
|
|
|||
|
|
#define HeapDebugBreak( _x_ ) {if (KdDebuggerEnabled) DbgBreakPoint();}
|
|||
|
|
|
|||
|
|
#else // #ifdef NTOS_KERNEL_ROUTINE
|
|||
|
|
|
|||
|
|
#define HeapDebugPrint( _x_ ) \
|
|||
|
|
{ \
|
|||
|
|
PLIST_ENTRY _Module; \
|
|||
|
|
PLDR_DATA_TABLE_ENTRY _Entry; \
|
|||
|
|
\
|
|||
|
|
_Module = NtCurrentPeb()->Ldr->InLoadOrderModuleList.Flink; \
|
|||
|
|
_Entry = CONTAINING_RECORD( _Module, \
|
|||
|
|
LDR_DATA_TABLE_ENTRY, \
|
|||
|
|
InLoadOrderLinks); \
|
|||
|
|
DbgPrint("HEAP[%wZ]: ", &_Entry->BaseDllName); \
|
|||
|
|
DbgPrint _x_; \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#define HeapDebugBreak( _x_ ) \
|
|||
|
|
{ \
|
|||
|
|
VOID RtlpBreakPointHeap( PVOID BadAddress ); \
|
|||
|
|
\
|
|||
|
|
RtlpBreakPointHeap( (_x_) ); \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#endif // #ifdef NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Virtual memory hook for virtual alloc functions
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#ifdef NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
#define RtlpHeapFreeVirtualMemory(P,A,S,F) \
|
|||
|
|
ZwFreeVirtualMemory(P,A,S,F)
|
|||
|
|
|
|||
|
|
#else // NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// The user mode call needs to call the secmem virtual free
|
|||
|
|
// as well to update the memory counters per heap
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlpHeapFreeVirtualMemory(P,A,S,F) \
|
|||
|
|
RtlpSecMemFreeVirtualMemory(P,A,S,F)
|
|||
|
|
|
|||
|
|
#endif // NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Implemented in heap.c
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
BOOLEAN
|
|||
|
|
RtlpInitializeHeapSegment (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN PHEAP_SEGMENT Segment,
|
|||
|
|
IN UCHAR SegmentIndex,
|
|||
|
|
IN ULONG Flags,
|
|||
|
|
IN PVOID BaseAddress,
|
|||
|
|
IN PVOID UnCommittedAddress,
|
|||
|
|
IN PVOID CommitLimitAddress
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
PHEAP_FREE_ENTRY
|
|||
|
|
RtlpCoalesceFreeBlocks (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN PHEAP_FREE_ENTRY FreeBlock,
|
|||
|
|
IN OUT PSIZE_T FreeSize,
|
|||
|
|
IN BOOLEAN RemoveFromFreeList
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpDeCommitFreeBlock (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN PHEAP_FREE_ENTRY FreeBlock,
|
|||
|
|
IN SIZE_T FreeSize
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpInsertFreeBlock (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN PHEAP_FREE_ENTRY FreeBlock,
|
|||
|
|
IN SIZE_T FreeSize
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
PHEAP_FREE_ENTRY
|
|||
|
|
RtlpFindAndCommitPages (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN PHEAP_SEGMENT Segment,
|
|||
|
|
IN OUT PSIZE_T Size,
|
|||
|
|
IN PVOID AddressWanted OPTIONAL
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
PVOID
|
|||
|
|
RtlAllocateHeapSlowly (
|
|||
|
|
IN PVOID HeapHandle,
|
|||
|
|
IN ULONG Flags,
|
|||
|
|
IN SIZE_T Size
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
BOOLEAN
|
|||
|
|
RtlFreeHeapSlowly (
|
|||
|
|
IN PVOID HeapHandle,
|
|||
|
|
IN ULONG Flags,
|
|||
|
|
IN PVOID BaseAddress
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
SIZE_T
|
|||
|
|
RtlpGetSizeOfBigBlock (
|
|||
|
|
IN PHEAP_ENTRY BusyBlock
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
PHEAP_ENTRY_EXTRA
|
|||
|
|
RtlpGetExtraStuffPointer (
|
|||
|
|
PHEAP_ENTRY BusyBlock
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
BOOLEAN
|
|||
|
|
RtlpCheckBusyBlockTail (
|
|||
|
|
IN PHEAP_ENTRY BusyBlock
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Implemented in heapdll.c
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpAddHeapToProcessList (
|
|||
|
|
IN PHEAP Heap
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpRemoveHeapFromProcessList (
|
|||
|
|
IN PHEAP Heap
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
PHEAP_FREE_ENTRY
|
|||
|
|
RtlpCoalesceHeap (
|
|||
|
|
IN PHEAP Heap
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
BOOLEAN
|
|||
|
|
RtlpCheckHeapSignature (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN PCHAR Caller
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlDetectHeapLeaks();
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Implemented in heapdbg.c
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
BOOLEAN
|
|||
|
|
RtlpValidateHeapEntry (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN PHEAP_ENTRY BusyBlock,
|
|||
|
|
IN PCHAR Reason
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
BOOLEAN
|
|||
|
|
RtlpValidateHeap (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN BOOLEAN AlwaysValidate
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpUpdateHeapListIndex (
|
|||
|
|
USHORT OldIndex,
|
|||
|
|
USHORT NewIndex
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
BOOLEAN
|
|||
|
|
RtlpValidateHeapHeaders(
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN BOOLEAN Recompute
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
|
|||
|
|
#ifndef NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Nondedicated free list optimization
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#if DBG
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Define HEAP_VALIDATE_INDEX to activate a validation of the index
|
|||
|
|
// after each operation with non-dedicated list
|
|||
|
|
// This is only for debug-test, to make sure the list and index is consistent
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
//#define HEAP_VALIDATE_INDEX
|
|||
|
|
|
|||
|
|
#endif // DBG
|
|||
|
|
|
|||
|
|
|
|||
|
|
#define HEAP_FRONT_LOOKASIDE 1
|
|||
|
|
#define HEAP_FRONT_LOWFRAGHEAP 2
|
|||
|
|
|
|||
|
|
#define RtlpGetLookasideHeap(H) \
|
|||
|
|
(((H)->FrontEndHeapType == HEAP_FRONT_LOOKASIDE) ? (H)->FrontEndHeap : NULL)
|
|||
|
|
|
|||
|
|
#define RtlpGetLowFragHeap(H) \
|
|||
|
|
(((H)->FrontEndHeapType == HEAP_FRONT_LOWFRAGHEAP) ? (H)->FrontEndHeap : NULL)
|
|||
|
|
|
|||
|
|
#define RtlpIsFrontHeapUnlocked(H) \
|
|||
|
|
((H)->FrontHeapLockCount == 0)
|
|||
|
|
|
|||
|
|
#define RtlpLockFrontHeap(H) \
|
|||
|
|
{ \
|
|||
|
|
(H)->FrontHeapLockCount += 1; \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#define RtlpUnlockFrontHeap(H) \
|
|||
|
|
{ \
|
|||
|
|
(H)->FrontHeapLockCount -= 1; \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
#define HEAP_INDEX_THRESHOLD 32
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Heap performance counter support
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define HEAP_OP_COUNT 2
|
|||
|
|
|
|||
|
|
#define HEAP_OP_ALLOC 0
|
|||
|
|
#define HEAP_OP_FREE 1
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// The time / per operation is measured ones at 16 operations
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define HEAP_SAMPLING_MASK 0x000001FF
|
|||
|
|
|
|||
|
|
#define HEAP_SAMPLING_COUNT 100
|
|||
|
|
|
|||
|
|
typedef struct _HEAP_PERF_DATA {
|
|||
|
|
|
|||
|
|
UINT64 CountFrequence;
|
|||
|
|
UINT64 OperationTime[HEAP_OP_COUNT];
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// The data bellow are only for sampling
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
ULONG Sequence;
|
|||
|
|
|
|||
|
|
UINT64 TempTime[HEAP_OP_COUNT];
|
|||
|
|
ULONG TempCount[HEAP_OP_COUNT];
|
|||
|
|
|
|||
|
|
} HEAP_PERF_DATA, *PHEAP_PERF_DATA;
|
|||
|
|
|
|||
|
|
#define HEAP_PERF_DECLARE_TIMER() \
|
|||
|
|
UINT64 _HeapPerfStartTimer, _HeapPerfEndTimer;
|
|||
|
|
|
|||
|
|
#define HEAP_PERF_START_TIMER(H) \
|
|||
|
|
{ \
|
|||
|
|
PHEAP_INDEX HeapIndex = (PHEAP_INDEX)(H)->LargeBlocksIndex; \
|
|||
|
|
if ( (HeapIndex != NULL) && \
|
|||
|
|
(!((HeapIndex->PerfData.Sequence++) & HEAP_SAMPLING_MASK)) ) { \
|
|||
|
|
\
|
|||
|
|
NtQueryPerformanceCounter( (PLARGE_INTEGER)&_HeapPerfStartTimer , NULL); \
|
|||
|
|
} else { \
|
|||
|
|
_HeapPerfStartTimer = 0; \
|
|||
|
|
} \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#define HEAP_PERF_STOP_TIMER(H,OP) \
|
|||
|
|
{ \
|
|||
|
|
if (_HeapPerfStartTimer) { \
|
|||
|
|
PHEAP_INDEX HeapIndex = (PHEAP_INDEX)(H)->LargeBlocksIndex; \
|
|||
|
|
\
|
|||
|
|
NtQueryPerformanceCounter( (PLARGE_INTEGER)&_HeapPerfEndTimer , NULL); \
|
|||
|
|
HeapIndex->PerfData.TempTime[OP] += (_HeapPerfEndTimer - _HeapPerfStartTimer); \
|
|||
|
|
\
|
|||
|
|
if ((HeapIndex->PerfData.TempCount[OP]++) >= HEAP_SAMPLING_COUNT) { \
|
|||
|
|
HeapIndex->PerfData.OperationTime[OP] = HeapIndex->PerfData.TempTime[OP] / (HeapIndex->PerfData.TempCount[OP] - 1); \
|
|||
|
|
\
|
|||
|
|
HeapIndex->PerfData.TempCount[OP] = 0; \
|
|||
|
|
HeapIndex->PerfData.TempTime[OP] = 0; \
|
|||
|
|
} \
|
|||
|
|
} \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#define RtlpRegisterOperation(H,S,Op) \
|
|||
|
|
{ \
|
|||
|
|
PHEAP_LOOKASIDE Lookaside; \
|
|||
|
|
\
|
|||
|
|
if ( (Lookaside = (PHEAP_LOOKASIDE)RtlpGetLookasideHeap(H)) ) { \
|
|||
|
|
\
|
|||
|
|
SIZE_T Index = (S) >> 10; \
|
|||
|
|
\
|
|||
|
|
if (Index >= HEAP_MAXIMUM_FREELISTS) { \
|
|||
|
|
\
|
|||
|
|
Index = HEAP_MAXIMUM_FREELISTS - 1; \
|
|||
|
|
} \
|
|||
|
|
\
|
|||
|
|
Lookaside[Index].Counters[(Op)] += 1; \
|
|||
|
|
} \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// The heap index structure
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
typedef struct _HEAP_INDEX {
|
|||
|
|
|
|||
|
|
ULONG ArraySize;
|
|||
|
|
ULONG VirtualMemorySize;
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// The timing counters are available only on heaps
|
|||
|
|
// with an index created
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
HEAP_PERF_DATA PerfData;
|
|||
|
|
|
|||
|
|
LONG LargeBlocksCacheDepth;
|
|||
|
|
LONG LargeBlocksCacheMaxDepth;
|
|||
|
|
LONG LargeBlocksCacheMinDepth;
|
|||
|
|
LONG LargeBlocksCacheSequence;
|
|||
|
|
|
|||
|
|
struct {
|
|||
|
|
|
|||
|
|
ULONG Committs;
|
|||
|
|
ULONG Decommitts;
|
|||
|
|
LONG LargestDepth;
|
|||
|
|
LONG LargestRequiredDepth;
|
|||
|
|
|
|||
|
|
} CacheStats;
|
|||
|
|
|
|||
|
|
union {
|
|||
|
|
|
|||
|
|
PULONG FreeListsInUseUlong;
|
|||
|
|
PUCHAR FreeListsInUseBytes;
|
|||
|
|
} u;
|
|||
|
|
|
|||
|
|
PHEAP_FREE_ENTRY * FreeListHints;
|
|||
|
|
|
|||
|
|
} HEAP_INDEX, *PHEAP_INDEX;
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Macro for setting a bit in the freelist vector to indicate entries are
|
|||
|
|
// present.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define SET_INDEX_BIT( HeapIndex, AllocIndex ) \
|
|||
|
|
{ \
|
|||
|
|
ULONG _Index_; \
|
|||
|
|
ULONG _Bit_; \
|
|||
|
|
\
|
|||
|
|
_Index_ = (AllocIndex) >> 3; \
|
|||
|
|
_Bit_ = (1 << ((AllocIndex) & 7)); \
|
|||
|
|
\
|
|||
|
|
(HeapIndex)->u.FreeListsInUseBytes[ _Index_ ] |= _Bit_; \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Macro for clearing a bit in the freelist vector to indicate entries are
|
|||
|
|
// not present.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define CLEAR_INDEX_BIT( HeapIndex, AllocIndex ) \
|
|||
|
|
{ \
|
|||
|
|
ULONG _Index_; \
|
|||
|
|
ULONG _Bit_; \
|
|||
|
|
\
|
|||
|
|
_Index_ = (AllocIndex) >> 3; \
|
|||
|
|
_Bit_ = (1 << ((AllocIndex) & 7)); \
|
|||
|
|
\
|
|||
|
|
(HeapIndex)->u.FreeListsInUseBytes[ _Index_ ] ^= _Bit_; \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpInitializeListIndex (
|
|||
|
|
IN PHEAP Heap
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
PLIST_ENTRY
|
|||
|
|
RtlpFindEntry (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN ULONG Size
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpUpdateIndexRemoveBlock (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN PHEAP_FREE_ENTRY FreeEntry
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpUpdateIndexInsertBlock (
|
|||
|
|
IN PHEAP Heap,
|
|||
|
|
IN PHEAP_FREE_ENTRY FreeEntry
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpFlushCacheContents (
|
|||
|
|
IN PHEAP Heap
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
extern LONG RtlpSequenceNumberTest;
|
|||
|
|
|
|||
|
|
#define RtlpCheckLargeCache(H) \
|
|||
|
|
{ \
|
|||
|
|
PHEAP_INDEX HeapIndex = (PHEAP_INDEX)(H)->LargeBlocksIndex; \
|
|||
|
|
if ((HeapIndex != NULL) && \
|
|||
|
|
(HeapIndex->LargeBlocksCacheSequence >= RtlpSequenceNumberTest)) { \
|
|||
|
|
\
|
|||
|
|
RtlpFlushCacheContents(Heap); \
|
|||
|
|
} \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpFlushLargestCacheBlock (
|
|||
|
|
IN PHEAP Heap
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
#ifdef HEAP_VALIDATE_INDEX
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// The validation code for index
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
BOOLEAN
|
|||
|
|
RtlpValidateNonDedicatedList (
|
|||
|
|
IN PHEAP Heap
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
|
|||
|
|
#else // HEAP_VALIDATE_INDEX
|
|||
|
|
|
|||
|
|
#define RtlpValidateNonDedicatedList(H)
|
|||
|
|
|
|||
|
|
#endif // HEAP_VALIDATE_INDEX
|
|||
|
|
|
|||
|
|
|
|||
|
|
#else // NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
#define HEAP_PERF_DECLARE_TIMER()
|
|||
|
|
|
|||
|
|
#define HEAP_PERF_START_TIMER(H)
|
|||
|
|
|
|||
|
|
#define HEAP_PERF_STOP_TIMER(H,Op)
|
|||
|
|
|
|||
|
|
#define RtlpRegisterOperation(H,S,Op)
|
|||
|
|
|
|||
|
|
|
|||
|
|
#define RtlpInitializeListIndex(H)
|
|||
|
|
|
|||
|
|
#define RtlpFindEntry(H,S) (NULL)
|
|||
|
|
|
|||
|
|
#define RtlpUpdateIndexRemoveBlock(H,F)
|
|||
|
|
|
|||
|
|
#define RtlpUpdateIndexInsertBlock(H,F)
|
|||
|
|
|
|||
|
|
#define RtlpCheckLargeCache(H)
|
|||
|
|
|
|||
|
|
#define RtlpValidateNonDedicatedList(H)
|
|||
|
|
|
|||
|
|
#endif // NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// An extra bitmap manipulation routine
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlFindFirstSetRightMember(Set) \
|
|||
|
|
(((Set) & 0xFFFF) ? \
|
|||
|
|
(((Set) & 0xFF) ? \
|
|||
|
|
RtlpBitsClearLow[(Set) & 0xFF] : \
|
|||
|
|
RtlpBitsClearLow[((Set) >> 8) & 0xFF] + 8) : \
|
|||
|
|
((((Set) >> 16) & 0xFF) ? \
|
|||
|
|
RtlpBitsClearLow[ ((Set) >> 16) & 0xFF] + 16 : \
|
|||
|
|
RtlpBitsClearLow[ (Set) >> 24] + 24) \
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Macro for setting a bit in the freelist vector to indicate entries are
|
|||
|
|
// present.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define SET_FREELIST_BIT( H, FB ) \
|
|||
|
|
{ \
|
|||
|
|
ULONG _Index_; \
|
|||
|
|
ULONG _Bit_; \
|
|||
|
|
\
|
|||
|
|
HEAPASSERT((FB)->Size < HEAP_MAXIMUM_FREELISTS); \
|
|||
|
|
\
|
|||
|
|
_Index_ = (FB)->Size >> 3; \
|
|||
|
|
_Bit_ = (1 << ((FB)->Size & 7)); \
|
|||
|
|
\
|
|||
|
|
HEAPASSERT(((H)->u.FreeListsInUseBytes[ _Index_ ] & _Bit_) == 0); \
|
|||
|
|
\
|
|||
|
|
(H)->u.FreeListsInUseBytes[ _Index_ ] |= _Bit_; \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Macro for clearing a bit in the freelist vector to indicate entries are
|
|||
|
|
// not present.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define CLEAR_FREELIST_BIT( H, FB ) \
|
|||
|
|
{ \
|
|||
|
|
ULONG _Index_; \
|
|||
|
|
ULONG _Bit_; \
|
|||
|
|
\
|
|||
|
|
HEAPASSERT((FB)->Size < HEAP_MAXIMUM_FREELISTS); \
|
|||
|
|
\
|
|||
|
|
_Index_ = (FB)->Size >> 3; \
|
|||
|
|
_Bit_ = (1 << ((FB)->Size & 7)); \
|
|||
|
|
\
|
|||
|
|
HEAPASSERT((H)->u.FreeListsInUseBytes[ _Index_ ] & _Bit_); \
|
|||
|
|
HEAPASSERT(IsListEmpty(&(H)->FreeLists[ (FB)->Size ])); \
|
|||
|
|
\
|
|||
|
|
(H)->u.FreeListsInUseBytes[ _Index_ ] ^= _Bit_; \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This macro inserts a free block into the appropriate free list including
|
|||
|
|
// the [0] index list with entry filling if necessary
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlpInsertFreeBlockDirect( H, FB, SIZE ) \
|
|||
|
|
{ \
|
|||
|
|
PLIST_ENTRY _HEAD, _NEXT; \
|
|||
|
|
PHEAP_FREE_ENTRY _FB1; \
|
|||
|
|
\
|
|||
|
|
HEAPASSERT((FB)->Size == (SIZE)); \
|
|||
|
|
(FB)->Flags &= ~(HEAP_ENTRY_FILL_PATTERN | \
|
|||
|
|
HEAP_ENTRY_EXTRA_PRESENT | \
|
|||
|
|
HEAP_ENTRY_BUSY); \
|
|||
|
|
\
|
|||
|
|
if ((H)->Flags & HEAP_FREE_CHECKING_ENABLED) { \
|
|||
|
|
\
|
|||
|
|
RtlFillMemoryUlong( (PCHAR)((FB) + 1), \
|
|||
|
|
((SIZE) << HEAP_GRANULARITY_SHIFT) - \
|
|||
|
|
sizeof( *(FB) ), \
|
|||
|
|
FREE_HEAP_FILL ); \
|
|||
|
|
\
|
|||
|
|
(FB)->Flags |= HEAP_ENTRY_FILL_PATTERN; \
|
|||
|
|
} \
|
|||
|
|
\
|
|||
|
|
if ((SIZE) < HEAP_MAXIMUM_FREELISTS) { \
|
|||
|
|
\
|
|||
|
|
_HEAD = &(H)->FreeLists[ (SIZE) ]; \
|
|||
|
|
\
|
|||
|
|
if (IsListEmpty(_HEAD)) { \
|
|||
|
|
\
|
|||
|
|
SET_FREELIST_BIT( H, FB ); \
|
|||
|
|
} \
|
|||
|
|
\
|
|||
|
|
} else { \
|
|||
|
|
\
|
|||
|
|
_HEAD = &(H)->FreeLists[ 0 ]; \
|
|||
|
|
_NEXT = (H)->LargeBlocksIndex ? \
|
|||
|
|
RtlpFindEntry(H, SIZE) : \
|
|||
|
|
_HEAD->Flink; \
|
|||
|
|
\
|
|||
|
|
while (_HEAD != _NEXT) { \
|
|||
|
|
\
|
|||
|
|
_FB1 = CONTAINING_RECORD( _NEXT, HEAP_FREE_ENTRY, FreeList ); \
|
|||
|
|
\
|
|||
|
|
if ((SIZE) <= _FB1->Size) { \
|
|||
|
|
\
|
|||
|
|
break; \
|
|||
|
|
\
|
|||
|
|
} else { \
|
|||
|
|
\
|
|||
|
|
_NEXT = _NEXT->Flink; \
|
|||
|
|
} \
|
|||
|
|
} \
|
|||
|
|
\
|
|||
|
|
_HEAD = _NEXT; \
|
|||
|
|
} \
|
|||
|
|
\
|
|||
|
|
InsertTailList( _HEAD, &(FB)->FreeList ); \
|
|||
|
|
RtlpUpdateIndexInsertBlock(H, FB); \
|
|||
|
|
RtlpValidateNonDedicatedList(H); \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This version of RtlpInsertFreeBlockDirect does no filling.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlpFastInsertFreeBlockDirect( H, FB, SIZE ) \
|
|||
|
|
{ \
|
|||
|
|
if ((SIZE) < HEAP_MAXIMUM_FREELISTS) { \
|
|||
|
|
\
|
|||
|
|
RtlpFastInsertDedicatedFreeBlockDirect( H, FB, SIZE ); \
|
|||
|
|
\
|
|||
|
|
} else { \
|
|||
|
|
\
|
|||
|
|
RtlpFastInsertNonDedicatedFreeBlockDirect( H, FB, SIZE ); \
|
|||
|
|
} \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This version of RtlpInsertFreeBlockDirect only works for dedicated free
|
|||
|
|
// lists and doesn't do any filling.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlpFastInsertDedicatedFreeBlockDirect( H, FB, SIZE ) \
|
|||
|
|
{ \
|
|||
|
|
PLIST_ENTRY _HEAD; \
|
|||
|
|
\
|
|||
|
|
HEAPASSERT((FB)->Size == (SIZE)); \
|
|||
|
|
\
|
|||
|
|
if (!((FB)->Flags & HEAP_ENTRY_LAST_ENTRY)) { \
|
|||
|
|
\
|
|||
|
|
HEAPASSERT(((PHEAP_ENTRY)(FB) + (SIZE))->PreviousSize == (SIZE)); \
|
|||
|
|
} \
|
|||
|
|
\
|
|||
|
|
(FB)->Flags &= HEAP_ENTRY_LAST_ENTRY; \
|
|||
|
|
\
|
|||
|
|
_HEAD = &(H)->FreeLists[ (SIZE) ]; \
|
|||
|
|
\
|
|||
|
|
if (IsListEmpty(_HEAD)) { \
|
|||
|
|
\
|
|||
|
|
SET_FREELIST_BIT( H, FB ); \
|
|||
|
|
} \
|
|||
|
|
\
|
|||
|
|
InsertTailList( _HEAD, &(FB)->FreeList ); \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This version of RtlpInsertFreeBlockDirect only works for nondedicated free
|
|||
|
|
// lists and doesn't do any filling.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlpFastInsertNonDedicatedFreeBlockDirect( H, FB, SIZE ) \
|
|||
|
|
{ \
|
|||
|
|
PLIST_ENTRY _HEAD, _NEXT; \
|
|||
|
|
PHEAP_FREE_ENTRY _FB1; \
|
|||
|
|
\
|
|||
|
|
HEAPASSERT((FB)->Size == (SIZE)); \
|
|||
|
|
\
|
|||
|
|
if (!((FB)->Flags & HEAP_ENTRY_LAST_ENTRY)) { \
|
|||
|
|
\
|
|||
|
|
HEAPASSERT(((PHEAP_ENTRY)(FB) + (SIZE))->PreviousSize == (SIZE)); \
|
|||
|
|
} \
|
|||
|
|
\
|
|||
|
|
(FB)->Flags &= (HEAP_ENTRY_LAST_ENTRY); \
|
|||
|
|
\
|
|||
|
|
_HEAD = &(H)->FreeLists[ 0 ]; \
|
|||
|
|
_NEXT = (H)->LargeBlocksIndex ? \
|
|||
|
|
RtlpFindEntry(H, SIZE) : \
|
|||
|
|
_HEAD->Flink; \
|
|||
|
|
\
|
|||
|
|
while (_HEAD != _NEXT) { \
|
|||
|
|
\
|
|||
|
|
_FB1 = CONTAINING_RECORD( _NEXT, HEAP_FREE_ENTRY, FreeList ); \
|
|||
|
|
\
|
|||
|
|
if ((SIZE) <= _FB1->Size) { \
|
|||
|
|
\
|
|||
|
|
break; \
|
|||
|
|
\
|
|||
|
|
} else { \
|
|||
|
|
\
|
|||
|
|
_NEXT = _NEXT->Flink; \
|
|||
|
|
} \
|
|||
|
|
} \
|
|||
|
|
\
|
|||
|
|
InsertTailList( _NEXT, &(FB)->FreeList ); \
|
|||
|
|
RtlpUpdateIndexInsertBlock(H, FB); \
|
|||
|
|
RtlpValidateNonDedicatedList(H); \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This macro removes a block from its free list with fill checking if
|
|||
|
|
// necessary
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlpRemoveFreeBlock( H, FB ) \
|
|||
|
|
{ \
|
|||
|
|
RtlpFastRemoveFreeBlock( H, FB ) \
|
|||
|
|
\
|
|||
|
|
if ((FB)->Flags & HEAP_ENTRY_FILL_PATTERN) { \
|
|||
|
|
\
|
|||
|
|
SIZE_T cb, cbEqual; \
|
|||
|
|
PVOID p; \
|
|||
|
|
\
|
|||
|
|
cb = ((FB)->Size << HEAP_GRANULARITY_SHIFT) - sizeof( *(FB) ); \
|
|||
|
|
\
|
|||
|
|
if ((FB)->Flags & HEAP_ENTRY_EXTRA_PRESENT && \
|
|||
|
|
cb > sizeof( HEAP_FREE_ENTRY_EXTRA )) { \
|
|||
|
|
\
|
|||
|
|
cb -= sizeof( HEAP_FREE_ENTRY_EXTRA ); \
|
|||
|
|
} \
|
|||
|
|
\
|
|||
|
|
cbEqual = RtlCompareMemoryUlong( (PCHAR)((FB) + 1), \
|
|||
|
|
cb, \
|
|||
|
|
FREE_HEAP_FILL ); \
|
|||
|
|
\
|
|||
|
|
if (cbEqual != cb) { \
|
|||
|
|
\
|
|||
|
|
HeapDebugPrint(( \
|
|||
|
|
"HEAP: Free Heap block %lx modified at %lx after it was freed\n", \
|
|||
|
|
(FB), \
|
|||
|
|
(PCHAR)((FB) + 1) + cbEqual )); \
|
|||
|
|
\
|
|||
|
|
HeapDebugBreak((FB)); \
|
|||
|
|
} \
|
|||
|
|
} \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This version of RtlpRemoveFreeBlock does no fill checking
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlpFastRemoveFreeBlock( H, FB ) \
|
|||
|
|
{ \
|
|||
|
|
PLIST_ENTRY _EX_Blink; \
|
|||
|
|
PLIST_ENTRY _EX_Flink; \
|
|||
|
|
RtlpUpdateIndexRemoveBlock(H, FB); \
|
|||
|
|
\
|
|||
|
|
_EX_Flink = (FB)->FreeList.Flink; \
|
|||
|
|
_EX_Blink = (FB)->FreeList.Blink; \
|
|||
|
|
\
|
|||
|
|
_EX_Blink->Flink = _EX_Flink; \
|
|||
|
|
_EX_Flink->Blink = _EX_Blink; \
|
|||
|
|
\
|
|||
|
|
if ((_EX_Flink == _EX_Blink) && \
|
|||
|
|
((FB)->Size < HEAP_MAXIMUM_FREELISTS)) { \
|
|||
|
|
\
|
|||
|
|
CLEAR_FREELIST_BIT( H, FB ); \
|
|||
|
|
} \
|
|||
|
|
RtlpValidateNonDedicatedList(H); \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This version of RtlpRemoveFreeBlock only works for dedicated free lists
|
|||
|
|
// (where we know that (FB)->Mask != 0) and doesn't do any fill checking
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlpFastRemoveDedicatedFreeBlock( H, FB ) \
|
|||
|
|
{ \
|
|||
|
|
PLIST_ENTRY _EX_Blink; \
|
|||
|
|
PLIST_ENTRY _EX_Flink; \
|
|||
|
|
\
|
|||
|
|
_EX_Flink = (FB)->FreeList.Flink; \
|
|||
|
|
_EX_Blink = (FB)->FreeList.Blink; \
|
|||
|
|
\
|
|||
|
|
_EX_Blink->Flink = _EX_Flink; \
|
|||
|
|
_EX_Flink->Blink = _EX_Blink; \
|
|||
|
|
\
|
|||
|
|
if (_EX_Flink == _EX_Blink) { \
|
|||
|
|
\
|
|||
|
|
CLEAR_FREELIST_BIT( H, FB ); \
|
|||
|
|
} \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This version of RtlpRemoveFreeBlock only works for dedicated free lists
|
|||
|
|
// (where we know that (FB)->Mask == 0) and doesn't do any fill checking
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlpFastRemoveNonDedicatedFreeBlock( H, FB ) \
|
|||
|
|
{ \
|
|||
|
|
RtlpUpdateIndexRemoveBlock(H, FB); \
|
|||
|
|
RemoveEntryList(&(FB)->FreeList); \
|
|||
|
|
RtlpValidateNonDedicatedList(H); \
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Heap tagging routines implemented in heapdll.c
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#if DBG
|
|||
|
|
|
|||
|
|
#define IS_HEAP_TAGGING_ENABLED() (TRUE)
|
|||
|
|
|
|||
|
|
#else
|
|||
|
|
|
|||
|
|
#define IS_HEAP_TAGGING_ENABLED() (RtlGetNtGlobalFlags() & FLG_HEAP_ENABLE_TAGGING)
|
|||
|
|
|
|||
|
|
#endif // DBG
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// ORDER IS IMPORTANT HERE...SEE RtlpUpdateTagEntry sources
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
typedef enum _HEAP_TAG_ACTION {
|
|||
|
|
|
|||
|
|
AllocationAction,
|
|||
|
|
VirtualAllocationAction,
|
|||
|
|
FreeAction,
|
|||
|
|
VirtualFreeAction,
|
|||
|
|
ReAllocationAction,
|
|||
|
|
VirtualReAllocationAction
|
|||
|
|
|
|||
|
|
} HEAP_TAG_ACTION;
|
|||
|
|
|
|||
|
|
PWSTR
|
|||
|
|
RtlpGetTagName (
|
|||
|
|
PHEAP Heap,
|
|||
|
|
USHORT TagIndex
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
USHORT
|
|||
|
|
RtlpUpdateTagEntry (
|
|||
|
|
PHEAP Heap,
|
|||
|
|
USHORT TagIndex,
|
|||
|
|
SIZE_T OldSize, // Only valid for ReAllocation and Free actions
|
|||
|
|
SIZE_T NewSize, // Only valid for ReAllocation and Allocation actions
|
|||
|
|
HEAP_TAG_ACTION Action
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpResetTags (
|
|||
|
|
PHEAP Heap
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpDestroyTags (
|
|||
|
|
PHEAP Heap
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Define heap lookaside list allocation functions.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
typedef struct _HEAP_LOOKASIDE {
|
|||
|
|
SLIST_HEADER ListHead;
|
|||
|
|
|
|||
|
|
USHORT Depth;
|
|||
|
|
USHORT MaximumDepth;
|
|||
|
|
|
|||
|
|
ULONG TotalAllocates;
|
|||
|
|
ULONG AllocateMisses;
|
|||
|
|
ULONG TotalFrees;
|
|||
|
|
ULONG FreeMisses;
|
|||
|
|
|
|||
|
|
ULONG LastTotalAllocates;
|
|||
|
|
ULONG LastAllocateMisses;
|
|||
|
|
|
|||
|
|
ULONG Counters[2];
|
|||
|
|
|
|||
|
|
} HEAP_LOOKASIDE, *PHEAP_LOOKASIDE;
|
|||
|
|
|
|||
|
|
NTKERNELAPI
|
|||
|
|
VOID
|
|||
|
|
RtlpInitializeHeapLookaside (
|
|||
|
|
IN PHEAP_LOOKASIDE Lookaside,
|
|||
|
|
IN USHORT Depth
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
NTKERNELAPI
|
|||
|
|
VOID
|
|||
|
|
RtlpDeleteHeapLookaside (
|
|||
|
|
IN PHEAP_LOOKASIDE Lookaside
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpAdjustHeapLookasideDepth (
|
|||
|
|
IN PHEAP_LOOKASIDE Lookaside
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
NTKERNELAPI
|
|||
|
|
PVOID
|
|||
|
|
RtlpAllocateFromHeapLookaside (
|
|||
|
|
IN PHEAP_LOOKASIDE Lookaside
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
NTKERNELAPI
|
|||
|
|
BOOLEAN
|
|||
|
|
RtlpFreeToHeapLookaside (
|
|||
|
|
IN PHEAP_LOOKASIDE Lookaside,
|
|||
|
|
IN PVOID Entry
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
#ifndef NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// Low Fragmentation Heap data structures and internal APIs
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// The memory barrier exists on IA64 only
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#if defined(_IA64_)
|
|||
|
|
|
|||
|
|
#define RtlMemoryBarrier() __mf ()
|
|||
|
|
|
|||
|
|
#else // #if defined(_IA64_)
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// On x86 and AMD64 ignore the memory barrier
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#define RtlMemoryBarrier()
|
|||
|
|
|
|||
|
|
#endif // #if defined(_IA64_)
|
|||
|
|
|
|||
|
|
extern ULONG RtlpDisableHeapLookaside;
|
|||
|
|
|
|||
|
|
#define HEAP_ENABLE_LOW_FRAG_HEAP 8
|
|||
|
|
|
|||
|
|
typedef struct _BLOCK_ENTRY {
|
|||
|
|
|
|||
|
|
HEAP_ENTRY;
|
|||
|
|
|
|||
|
|
USHORT LinkOffset;
|
|||
|
|
USHORT Reserved2;
|
|||
|
|
|
|||
|
|
} BLOCK_ENTRY, *PBLOCK_ENTRY;
|
|||
|
|
|
|||
|
|
|
|||
|
|
typedef struct _INTERLOCK_SEQ {
|
|||
|
|
|
|||
|
|
union {
|
|||
|
|
|
|||
|
|
struct {
|
|||
|
|
|
|||
|
|
union {
|
|||
|
|
|
|||
|
|
struct {
|
|||
|
|
|
|||
|
|
USHORT Depth;
|
|||
|
|
USHORT FreeEntryOffset;
|
|||
|
|
};
|
|||
|
|
volatile ULONG OffsetAndDepth;
|
|||
|
|
};
|
|||
|
|
volatile ULONG Sequence;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
volatile LONGLONG Exchg;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
} INTERLOCK_SEQ, *PINTERLOCK_SEQ;
|
|||
|
|
|
|||
|
|
struct _HEAP_USERDATA_HEADER;
|
|||
|
|
|
|||
|
|
typedef struct _HEAP_SUBSEGMENT {
|
|||
|
|
|
|||
|
|
PVOID Bucket;
|
|||
|
|
|
|||
|
|
volatile struct _HEAP_USERDATA_HEADER * UserBlocks;
|
|||
|
|
|
|||
|
|
INTERLOCK_SEQ AggregateExchg;
|
|||
|
|
|
|||
|
|
union {
|
|||
|
|
|
|||
|
|
struct {
|
|||
|
|
USHORT BlockSize;
|
|||
|
|
USHORT FreeThreshold;
|
|||
|
|
USHORT BlockCount;
|
|||
|
|
UCHAR SizeIndex;
|
|||
|
|
UCHAR AffinityIndex;
|
|||
|
|
};
|
|||
|
|
ULONG Alignment[2];
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
SINGLE_LIST_ENTRY SFreeListEntry;
|
|||
|
|
volatile ULONG Lock;
|
|||
|
|
|
|||
|
|
} HEAP_SUBSEGMENT, *PHEAP_SUBSEGMENT;
|
|||
|
|
|
|||
|
|
typedef struct _HEAP_USERDATA_HEADER {
|
|||
|
|
|
|||
|
|
union {
|
|||
|
|
|
|||
|
|
SINGLE_LIST_ENTRY SFreeListEntry;
|
|||
|
|
PHEAP_SUBSEGMENT SubSegment;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
PVOID HeapHandle;
|
|||
|
|
|
|||
|
|
ULONG_PTR SizeIndex;
|
|||
|
|
ULONG_PTR Signature;
|
|||
|
|
|
|||
|
|
} HEAP_USERDATA_HEADER, *PHEAP_USERDATA_HEADER;
|
|||
|
|
|
|||
|
|
#define HEAP_LFH_INDEX 0xFF
|
|||
|
|
#define HEAP_LFH_IN_CONVERSION 0xFE
|
|||
|
|
|
|||
|
|
#define HEAP_NO_CACHE_BLOCK 0x800000
|
|||
|
|
#define HEAP_LARGEST_LFH_BLOCK 0x4000
|
|||
|
|
#define HEAP_LFH_USER_SIGNATURE 0xF0E0D0C0
|
|||
|
|
|
|||
|
|
#ifdef DISABLE_REGISTRY_TEST_HOOKS
|
|||
|
|
|
|||
|
|
#define RtlpIsLowFragHeapEnabled() FALSE
|
|||
|
|
|
|||
|
|
#else //DISABLE_REGISTRY_TEST_HOOKS
|
|||
|
|
|
|||
|
|
#define RtlpIsLowFragHeapEnabled() \
|
|||
|
|
((RtlpDisableHeapLookaside & HEAP_ENABLE_LOW_FRAG_HEAP) != 0)
|
|||
|
|
|
|||
|
|
#endif //DISABLE_REGISTRY_TEST_HOOKS
|
|||
|
|
|
|||
|
|
|
|||
|
|
ULONG
|
|||
|
|
FORCEINLINE
|
|||
|
|
RtlpGetAllocationUnits(
|
|||
|
|
PHEAP Heap,
|
|||
|
|
PHEAP_ENTRY Block
|
|||
|
|
)
|
|||
|
|
{
|
|||
|
|
|
|||
|
|
PHEAP_SUBSEGMENT SubSegment = (PHEAP_SUBSEGMENT)Block->SubSegment;
|
|||
|
|
|
|||
|
|
RtlMemoryBarrier();
|
|||
|
|
|
|||
|
|
if (Block->SegmentIndex == HEAP_LFH_INDEX) {
|
|||
|
|
|
|||
|
|
ULONG ReturnSize = *((volatile USHORT *)&SubSegment->BlockSize);
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// ISSUE: Workaround the x86 compiler bug which eliminates the second test
|
|||
|
|
// if (Block->SegmentIndex == HEAP_LFH_INDEX)
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
#if !defined(_WIN64)
|
|||
|
|
_asm {
|
|||
|
|
|
|||
|
|
nop
|
|||
|
|
}
|
|||
|
|
#endif
|
|||
|
|
|
|||
|
|
RtlMemoryBarrier();
|
|||
|
|
|
|||
|
|
if (Block->SegmentIndex == HEAP_LFH_INDEX) {
|
|||
|
|
|
|||
|
|
return ReturnSize;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (Block->SegmentIndex == HEAP_LFH_IN_CONVERSION) {
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This should be a very rare case when this query is
|
|||
|
|
// done in the small window when the conversion code sets the
|
|||
|
|
// Size & PrevSize fields.
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
RtlLockHeap(Heap);
|
|||
|
|
RtlUnlockHeap(Heap);
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// This makes sure the conversion completed, so we can grab the
|
|||
|
|
// block size like for a regular block
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return Block->Size;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
FORCEINLINE
|
|||
|
|
RtlpSetUnusedBytes(PHEAP Heap, PHEAP_ENTRY Block, SIZE_T UnusedBytes)
|
|||
|
|
{
|
|||
|
|
if (UnusedBytes < 0xff) {
|
|||
|
|
|
|||
|
|
Block->UnusedBytes = (UCHAR)(UnusedBytes);
|
|||
|
|
|
|||
|
|
} else {
|
|||
|
|
|
|||
|
|
PSIZE_T UnusedBytesULong = (PSIZE_T)(Block + RtlpGetAllocationUnits(Heap, Block));
|
|||
|
|
|
|||
|
|
UnusedBytesULong -= 1;
|
|||
|
|
Block->UnusedBytes = 0xff;
|
|||
|
|
*UnusedBytesULong = UnusedBytes;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
SIZE_T
|
|||
|
|
FORCEINLINE
|
|||
|
|
RtlpGetUnusedBytes(PHEAP Heap, PHEAP_ENTRY Block)
|
|||
|
|
{
|
|||
|
|
if (Block->UnusedBytes < 0xff) {
|
|||
|
|
|
|||
|
|
return Block->UnusedBytes;
|
|||
|
|
|
|||
|
|
} else {
|
|||
|
|
|
|||
|
|
PSIZE_T UnusedBytesULong = (PSIZE_T)(Block + RtlpGetAllocationUnits(Heap, Block));
|
|||
|
|
UnusedBytesULong -= 1;
|
|||
|
|
|
|||
|
|
return (*UnusedBytesULong);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
RtlpInitializeLowFragHeapManager();
|
|||
|
|
|
|||
|
|
HANDLE
|
|||
|
|
FASTCALL
|
|||
|
|
RtlpCreateLowFragHeap(
|
|||
|
|
HANDLE Heap
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
FASTCALL
|
|||
|
|
RtlpDestroyLowFragHeap(
|
|||
|
|
HANDLE LowFragHeapHandle
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
PVOID
|
|||
|
|
FASTCALL
|
|||
|
|
RtlpLowFragHeapAlloc(
|
|||
|
|
HANDLE LowFragHeapHandle,
|
|||
|
|
SIZE_T BlockSize
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
BOOLEAN
|
|||
|
|
FASTCALL
|
|||
|
|
RtlpLowFragHeapFree(
|
|||
|
|
HANDLE LowFragHeapHandle,
|
|||
|
|
PVOID p
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
NTSTATUS
|
|||
|
|
RtlpActivateLowFragmentationHeap(
|
|||
|
|
IN PVOID HeapHandle
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
#else // NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
//
|
|||
|
|
// The kernel mode heap does not ajdust the heap granularity
|
|||
|
|
// therefore the unused bytes always fit the UCHAR.
|
|||
|
|
// No need to check for overflow here
|
|||
|
|
//
|
|||
|
|
|
|||
|
|
ULONG
|
|||
|
|
FORCEINLINE
|
|||
|
|
RtlpGetAllocationUnits(
|
|||
|
|
PHEAP Heap,
|
|||
|
|
PHEAP_ENTRY Block
|
|||
|
|
)
|
|||
|
|
{
|
|||
|
|
return Block->Size;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
VOID
|
|||
|
|
FORCEINLINE
|
|||
|
|
RtlpSetUnusedBytes(PHEAP Heap, PHEAP_ENTRY Block, SIZE_T UnusedBytes)
|
|||
|
|
{
|
|||
|
|
Block->UnusedBytes = (UCHAR)(UnusedBytes);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
SIZE_T
|
|||
|
|
FORCEINLINE
|
|||
|
|
RtlpGetUnusedBytes(PHEAP Heap, PHEAP_ENTRY Block)
|
|||
|
|
{
|
|||
|
|
return Block->UnusedBytes;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#endif // NTOS_KERNEL_RUNTIME
|
|||
|
|
|
|||
|
|
#endif // _RTL_HEAP_PRIVATE_
|