544 lines
13 KiB
C
544 lines
13 KiB
C
/*++
|
||
|
||
Copyright (c) 1993 Microsoft Corporation
|
||
|
||
Module Name:
|
||
|
||
kiinit.c
|
||
|
||
Abstract:
|
||
|
||
This module implements architecture independent kernel initialization.
|
||
|
||
Author:
|
||
|
||
David N. Cutler 11-May-1993
|
||
|
||
Environment:
|
||
|
||
Kernel mode only.
|
||
|
||
Revision History:
|
||
|
||
--*/
|
||
|
||
#include "ki.h"
|
||
|
||
//
|
||
// External data.
|
||
//
|
||
|
||
extern KSPIN_LOCK AfdWorkQueueSpinLock;
|
||
extern KSPIN_LOCK CcBcbSpinLock;
|
||
extern KSPIN_LOCK CcMasterSpinLock;
|
||
extern KSPIN_LOCK CcVacbSpinLock;
|
||
extern KSPIN_LOCK CcWorkQueueSpinLock;
|
||
extern KSPIN_LOCK IopCancelSpinLock;
|
||
extern KSPIN_LOCK IopCompletionLock;
|
||
extern KSPIN_LOCK IopDatabaseLock;
|
||
extern KSPIN_LOCK IopVpbSpinLock;
|
||
extern KSPIN_LOCK KiContextSwapLock;
|
||
extern KSPIN_LOCK KiDispatcherLock;
|
||
extern KSPIN_LOCK NtfsStructLock;
|
||
extern KSPIN_LOCK MmPfnLock;
|
||
extern KSPIN_LOCK NonPagedPoolLock;
|
||
extern KSPIN_LOCK MmSystemSpaceLock;
|
||
|
||
#if DBG && !defined(_X86_)
|
||
extern KSPIN_LOCK KipGlobalAlignmentDatabaseLock;
|
||
#endif
|
||
//
|
||
//
|
||
// The following exist to allow testing of NUMA support.
|
||
//
|
||
|
||
ULONG KeVerifyNumaPageShift;
|
||
ULONG KeVerifyNumaAffinityShift;
|
||
ULONG KeVerifyNumaNodeCount;
|
||
ULONG KeVerifyNumaAffinity;
|
||
ULONG KeVerifyNumaPageMask;
|
||
|
||
// End numa test support variables.
|
||
|
||
|
||
//
|
||
// Put all code for kernel initialization in the INIT section. It will be
|
||
// deallocated by memory management when phase 1 initialization is completed.
|
||
//
|
||
|
||
#if defined(ALLOC_PRAGMA)
|
||
|
||
#pragma alloc_text(INIT, KeInitSystem)
|
||
#pragma alloc_text(INIT, KiInitQueuedSpinLocks)
|
||
#pragma alloc_text(INIT, KiInitSystem)
|
||
#pragma alloc_text(INIT, KiComputeReciprocal)
|
||
#pragma alloc_text(INIT, KeNumaInitialize)
|
||
|
||
#endif
|
||
|
||
BOOLEAN
|
||
KeInitSystem (
|
||
VOID
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function initializes executive structures implemented by the
|
||
kernel.
|
||
|
||
N.B. This function is only called during phase 1 initialization.
|
||
|
||
Arguments:
|
||
|
||
None.
|
||
|
||
Return Value:
|
||
|
||
A value of TRUE is returned if initialization is successful. Otherwise,
|
||
a value of FALSE is returned.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
//
|
||
// Perform platform dependent initialization.
|
||
//
|
||
|
||
return KiInitMachineDependent();
|
||
}
|
||
|
||
VOID
|
||
KiInitQueuedSpinLocks (
|
||
PKPRCB Prcb,
|
||
ULONG Number
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function initializes the queued spinlock structures in the per
|
||
processor PRCB. This function is called once for each processor as
|
||
it is initialized in an MP system.
|
||
|
||
Arguments:
|
||
|
||
Prcb - Supplies a pointer to a PRCB.
|
||
|
||
Number - Supplies the number of respective processor.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
//
|
||
// Initialize queued spinlock structures.
|
||
//
|
||
|
||
Prcb->LockQueue[LockQueueDispatcherLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueDispatcherLock].Lock = &KiDispatcherLock;
|
||
|
||
Prcb->LockQueue[LockQueueContextSwapLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueContextSwapLock].Lock = &KiContextSwapLock;
|
||
|
||
Prcb->LockQueue[LockQueuePfnLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueuePfnLock].Lock = &MmPfnLock;
|
||
|
||
Prcb->LockQueue[LockQueueSystemSpaceLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueSystemSpaceLock].Lock = &MmSystemSpaceLock;
|
||
|
||
Prcb->LockQueue[LockQueueBcbLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueBcbLock].Lock = &CcBcbSpinLock;
|
||
|
||
Prcb->LockQueue[LockQueueMasterLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueMasterLock].Lock = &CcMasterSpinLock;
|
||
|
||
Prcb->LockQueue[LockQueueVacbLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueVacbLock].Lock = &CcVacbSpinLock;
|
||
|
||
Prcb->LockQueue[LockQueueWorkQueueLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueWorkQueueLock].Lock = &CcWorkQueueSpinLock;
|
||
|
||
Prcb->LockQueue[LockQueueNonPagedPoolLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueNonPagedPoolLock].Lock = &NonPagedPoolLock;
|
||
|
||
Prcb->LockQueue[LockQueueIoCancelLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueIoCancelLock].Lock = &IopCancelSpinLock;
|
||
|
||
Prcb->LockQueue[LockQueueIoVpbLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueIoVpbLock].Lock = &IopVpbSpinLock;
|
||
|
||
Prcb->LockQueue[LockQueueIoDatabaseLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueIoDatabaseLock].Lock = &IopDatabaseLock;
|
||
|
||
Prcb->LockQueue[LockQueueIoCompletionLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueIoCompletionLock].Lock = &IopCompletionLock;
|
||
|
||
Prcb->LockQueue[LockQueueNtfsStructLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueNtfsStructLock].Lock = &NtfsStructLock;
|
||
|
||
Prcb->LockQueue[LockQueueAfdWorkQueueLock].Next = NULL;
|
||
Prcb->LockQueue[LockQueueAfdWorkQueueLock].Lock = &AfdWorkQueueSpinLock;
|
||
|
||
//
|
||
// If this is processor zero, then also initialize the queued spin lock
|
||
// home address.
|
||
//
|
||
|
||
if (Number == 0) {
|
||
KeInitializeSpinLock(&KiContextSwapLock);
|
||
KeInitializeSpinLock(&KiDispatcherLock);
|
||
KeInitializeSpinLock(&MmPfnLock);
|
||
KeInitializeSpinLock(&MmSystemSpaceLock);
|
||
KeInitializeSpinLock(&CcBcbSpinLock);
|
||
KeInitializeSpinLock(&CcMasterSpinLock);
|
||
KeInitializeSpinLock(&CcVacbSpinLock);
|
||
KeInitializeSpinLock(&CcWorkQueueSpinLock);
|
||
KeInitializeSpinLock(&IopCancelSpinLock);
|
||
KeInitializeSpinLock(&IopCompletionLock);
|
||
KeInitializeSpinLock(&IopDatabaseLock);
|
||
KeInitializeSpinLock(&IopVpbSpinLock);
|
||
KeInitializeSpinLock(&NonPagedPoolLock);
|
||
KeInitializeSpinLock(&NtfsStructLock);
|
||
KeInitializeSpinLock(&AfdWorkQueueSpinLock);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
VOID
|
||
KiInitSystem (
|
||
VOID
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function initializes architecture independent kernel structures.
|
||
|
||
Arguments:
|
||
|
||
None.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
ULONG Index;
|
||
|
||
//
|
||
// Initialize dispatcher ready queue listheads.
|
||
//
|
||
|
||
for (Index = 0; Index < MAXIMUM_PRIORITY; Index += 1) {
|
||
InitializeListHead(&KiDispatcherReadyListHead[Index]);
|
||
}
|
||
|
||
//
|
||
// Initialize bug check callback listhead and spinlock.
|
||
//
|
||
|
||
InitializeListHead(&KeBugCheckCallbackListHead);
|
||
InitializeListHead(&KeBugCheckReasonCallbackListHead);
|
||
KeInitializeSpinLock(&KeBugCheckCallbackLock);
|
||
|
||
//
|
||
// Initialize the timer expiration DPC object.
|
||
//
|
||
|
||
KeInitializeDpc(&KiTimerExpireDpc,
|
||
(PKDEFERRED_ROUTINE)KiTimerExpiration, NIL);
|
||
|
||
//
|
||
// Initialize the profile listhead and profile locks
|
||
//
|
||
|
||
KeInitializeSpinLock(&KiProfileLock);
|
||
InitializeListHead(&KiProfileListHead);
|
||
|
||
#if DBG && !defined(_X86_)
|
||
|
||
//
|
||
// Initialize the global alignment fault database lock
|
||
//
|
||
|
||
KeInitializeSpinLock(&KipGlobalAlignmentDatabaseLock);
|
||
#endif
|
||
|
||
//
|
||
// Initialize the active profile source listhead.
|
||
//
|
||
|
||
InitializeListHead(&KiProfileSourceListHead);
|
||
|
||
//
|
||
// Initialize the timer table, the timer completion listhead, and the
|
||
// timer completion DPC.
|
||
//
|
||
|
||
for (Index = 0; Index < TIMER_TABLE_SIZE; Index += 1) {
|
||
InitializeListHead(&KiTimerTableListHead[Index]);
|
||
}
|
||
|
||
//
|
||
// Initialize the swap event, the process inswap listhead, the
|
||
// process outswap listhead, the kernel stack inswap listhead,
|
||
// the wait in listhead, and the wait out listhead.
|
||
//
|
||
|
||
KeInitializeEvent(&KiSwapEvent,
|
||
SynchronizationEvent,
|
||
FALSE);
|
||
|
||
KiProcessInSwapListHead.Next = NULL;
|
||
KiProcessOutSwapListHead.Next = NULL;
|
||
KiStackInSwapListHead.Next = NULL;
|
||
InitializeListHead(&KiWaitListHead);
|
||
|
||
//
|
||
// Initialize the system service descriptor table.
|
||
//
|
||
|
||
KeServiceDescriptorTable[0].Base = &KiServiceTable[0];
|
||
KeServiceDescriptorTable[0].Count = NULL;
|
||
KeServiceDescriptorTable[0].Limit = KiServiceLimit;
|
||
|
||
//
|
||
// The global pointer associated with the table base is placed just
|
||
// before the service table on the ia64.
|
||
//
|
||
|
||
#if defined(_IA64_)
|
||
|
||
KeServiceDescriptorTable[0].TableBaseGpOffset =
|
||
(LONG)(*(KiServiceTable-1) - (ULONG_PTR)KiServiceTable);
|
||
|
||
#endif
|
||
|
||
KeServiceDescriptorTable[0].Number = &KiArgumentTable[0];
|
||
for (Index = 1; Index < NUMBER_SERVICE_TABLES; Index += 1) {
|
||
KeServiceDescriptorTable[Index].Limit = 0;
|
||
}
|
||
|
||
//
|
||
// Copy the system service descriptor table to the shadow table
|
||
// which is used to record the Win32 system services.
|
||
//
|
||
|
||
RtlCopyMemory(KeServiceDescriptorTableShadow,
|
||
KeServiceDescriptorTable,
|
||
sizeof(KeServiceDescriptorTable));
|
||
|
||
//
|
||
// Initialize call performance data structures.
|
||
//
|
||
|
||
#if defined(_COLLECT_FLUSH_SINGLE_CALLDATA_)
|
||
|
||
ExInitializeCallData(&KiFlushSingleCallData);
|
||
|
||
#endif
|
||
|
||
#if defined(_COLLECT_SET_EVENT_CALLDATA_)
|
||
|
||
ExInitializeCallData(&KiSetEventCallData);
|
||
|
||
#endif
|
||
|
||
#if defined(_COLLECT_WAIT_SINGLE_CALLDATA_)
|
||
|
||
ExInitializeCallData(&KiWaitSingleCallData);
|
||
|
||
#endif
|
||
|
||
return;
|
||
}
|
||
|
||
LARGE_INTEGER
|
||
KiComputeReciprocal (
|
||
IN LONG Divisor,
|
||
OUT PCCHAR Shift
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
This function computes the large integer reciprocal of the specified
|
||
value.
|
||
|
||
Arguments:
|
||
|
||
Divisor - Supplies the value for which the large integer reciprocal is
|
||
computed.
|
||
|
||
Shift - Supplies a pointer to a variable that receives the computed
|
||
shift count.
|
||
|
||
Return Value:
|
||
|
||
The large integer reciprocal is returned as the fucntion value.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
LARGE_INTEGER Fraction;
|
||
LONG NumberBits;
|
||
LONG Remainder;
|
||
|
||
//
|
||
// Compute the large integer reciprocal of the specified value.
|
||
//
|
||
|
||
NumberBits = 0;
|
||
Remainder = 1;
|
||
Fraction.LowPart = 0;
|
||
Fraction.HighPart = 0;
|
||
while (Fraction.HighPart >= 0) {
|
||
NumberBits += 1;
|
||
Fraction.HighPart = (Fraction.HighPart << 1) | (Fraction.LowPart >> 31);
|
||
Fraction.LowPart <<= 1;
|
||
Remainder <<= 1;
|
||
if (Remainder >= Divisor) {
|
||
Remainder -= Divisor;
|
||
Fraction.LowPart |= 1;
|
||
}
|
||
}
|
||
|
||
if (Remainder != 0) {
|
||
if ((Fraction.LowPart == 0xffffffff) && (Fraction.HighPart == 0xffffffff)) {
|
||
Fraction.LowPart = 0;
|
||
Fraction.HighPart = 0x80000000;
|
||
NumberBits -= 1;
|
||
|
||
} else {
|
||
if (Fraction.LowPart == 0xffffffff) {
|
||
Fraction.LowPart = 0;
|
||
Fraction.HighPart += 1;
|
||
|
||
} else {
|
||
Fraction.LowPart += 1;
|
||
}
|
||
}
|
||
}
|
||
|
||
//
|
||
// Compute the shift count value and return the reciprocal fraction.
|
||
//
|
||
|
||
*Shift = (CCHAR)(NumberBits - 64);
|
||
return Fraction;
|
||
}
|
||
|
||
VOID
|
||
KeNumaInitialize (
|
||
VOID
|
||
)
|
||
|
||
/*++
|
||
|
||
Routine Description:
|
||
|
||
Initialize ntos kernel structures needed to support NUMA.
|
||
|
||
Arguments:
|
||
|
||
None.
|
||
|
||
Return Value:
|
||
|
||
None.
|
||
|
||
--*/
|
||
|
||
{
|
||
|
||
#if defined(KE_MULTINODE)
|
||
|
||
NTSTATUS Status;
|
||
HAL_NUMA_TOPOLOGY_INTERFACE HalNumaInfo;
|
||
ULONG ReturnedLength;
|
||
|
||
extern PHALNUMAQUERYPROCESSORNODE KiQueryProcessorNode;
|
||
extern PHALNUMAPAGETONODE MmPageToNode;
|
||
|
||
//
|
||
// Mega Kludge: For testing purposes on NON-Numa systems,
|
||
// we can have non-numa MP systems report a NUMA configuration.
|
||
//
|
||
// Pass the information obtained from the registry to the HAL
|
||
// in the return info buffer.
|
||
//
|
||
|
||
if (KeVerifyNumaNodeCount &&
|
||
(KeVerifyNumaNodeCount < 8) &&
|
||
KeVerifyNumaAffinity &&
|
||
KeVerifyNumaAffinityShift &&
|
||
(KeVerifyNumaAffinityShift < 32) &&
|
||
KeVerifyNumaPageMask &&
|
||
(KeVerifyNumaPageMask < KeVerifyNumaNodeCount)&&
|
||
KeVerifyNumaPageShift &&
|
||
(KeVerifyNumaPageShift < 32) ) {
|
||
|
||
struct {
|
||
ULONG Nodes:3;
|
||
ULONG AffinityShift:6;
|
||
ULONG PageShift:6;
|
||
ULONG Signature:17;
|
||
ULONG Affinity;
|
||
ULONG Mask;
|
||
} Fake;
|
||
|
||
C_ASSERT(sizeof(Fake) <= sizeof(HalNumaInfo));
|
||
|
||
Fake.Signature = 0x15a5a;
|
||
Fake.PageShift = KeVerifyNumaPageShift;
|
||
Fake.AffinityShift = KeVerifyNumaAffinityShift;
|
||
Fake.Nodes = KeVerifyNumaNodeCount;
|
||
Fake.Affinity = KeVerifyNumaAffinity;
|
||
Fake.Mask = KeVerifyNumaPageMask;
|
||
|
||
RtlCopyMemory(&HalNumaInfo, &Fake, sizeof(Fake));
|
||
}
|
||
|
||
//
|
||
// End Mega Kludge.
|
||
//
|
||
|
||
Status = HalQuerySystemInformation (HalNumaTopologyInterface,
|
||
sizeof(HalNumaInfo),
|
||
&HalNumaInfo,
|
||
&ReturnedLength);
|
||
|
||
if (NT_SUCCESS(Status)) {
|
||
|
||
ASSERT (ReturnedLength == sizeof(HalNumaInfo));
|
||
ASSERT (HalNumaInfo.NumberOfNodes <= MAXIMUM_CCNUMA_NODES);
|
||
ASSERT (HalNumaInfo.QueryProcessorNode);
|
||
ASSERT (HalNumaInfo.PageToNode);
|
||
|
||
if (HalNumaInfo.NumberOfNodes > 1) {
|
||
KeNumberNodes = (UCHAR)HalNumaInfo.NumberOfNodes;
|
||
MmPageToNode = HalNumaInfo.PageToNode;
|
||
KiQueryProcessorNode = HalNumaInfo.QueryProcessorNode;
|
||
}
|
||
}
|
||
|
||
|
||
#endif
|
||
|
||
}
|