2196 lines
63 KiB
NASM
2196 lines
63 KiB
NASM
title "Context Swap"
|
|
;++
|
|
;
|
|
; Copyright (c) 1989, 2000 Microsoft Corporation
|
|
;
|
|
; Module Name:
|
|
;
|
|
; ctxswap.asm
|
|
;
|
|
; Abstract:
|
|
;
|
|
; This module implements the code necessary to field the dispatch
|
|
; interrupt and to perform kernel initiated context switching.
|
|
;
|
|
; Author:
|
|
;
|
|
; Shie-Lin Tzong (shielint) 14-Jan-1990
|
|
;
|
|
; Environment:
|
|
;
|
|
; Kernel mode only.
|
|
;
|
|
; Revision History:
|
|
;
|
|
; 22-feb-90 bryanwi
|
|
; write actual swap context procedure
|
|
;
|
|
;--
|
|
|
|
.586p
|
|
.xlist
|
|
include ks386.inc
|
|
include i386\kimacro.inc
|
|
include mac386.inc
|
|
include callconv.inc
|
|
FPOFRAME macro a, b
|
|
.FPO ( a, b, 0, 0, 0, 0 )
|
|
endm
|
|
.list
|
|
|
|
EXTRNP KeAcquireQueuedSpinLockAtDpcLevel,1,,FASTCALL
|
|
EXTRNP KeReleaseQueuedSpinLockFromDpcLevel,1,,FASTCALL
|
|
EXTRNP KeTryToAcquireQueuedSpinLockAtRaisedIrql,1,,FASTCALL
|
|
EXTRNP KeAcquireQueuedSpinLockRaiseToSynch,1,,FASTCALL
|
|
|
|
EXTRNP KfAcquireSpinLock,1,IMPORT,FASTCALL
|
|
|
|
EXTRNP HalClearSoftwareInterrupt,1,IMPORT,FASTCALL
|
|
EXTRNP HalRequestSoftwareInterrupt,1,IMPORT,FASTCALL
|
|
EXTRNP KiActivateWaiterQueue,1,,FASTCALL
|
|
EXTRNP KiReadyThread,1,,FASTCALL
|
|
EXTRNP KiWaitTest,2,,FASTCALL
|
|
EXTRNP KfLowerIrql,1,IMPORT,FASTCALL
|
|
EXTRNP KfRaiseIrql,1,IMPORT,FASTCALL
|
|
EXTRNP _KeGetCurrentIrql,0,IMPORT
|
|
EXTRNP ___KeGetCurrentThread,0
|
|
EXTRNP _KiDeliverApc,3
|
|
EXTRNP _KiQuantumEnd,0
|
|
EXTRNP _KeBugCheckEx,5
|
|
EXTRNP _KeBugCheck,1
|
|
|
|
extrn _KiTrap13:PROC
|
|
extrn _KeI386FxsrPresent:BYTE
|
|
extrn _KiDispatcherLock:DWORD
|
|
extrn _KeFeatureBits:DWORD
|
|
extrn _KeThreadSwitchCounters:DWORD
|
|
|
|
extrn __imp_@KfLowerIrql@4:DWORD
|
|
|
|
extrn _KiDispatcherReadyListHead:DWORD
|
|
extrn _KiIdleSummary:DWORD
|
|
extrn _KiIdleSMTSummary:DWORD
|
|
extrn _KiReadySummary:DWORD
|
|
extrn _PPerfGlobalGroupMask:DWORD
|
|
|
|
EXTRNP WmiTraceContextSwap,2,,FASTCALL
|
|
EXTRNP PerfInfoLogDpc,3,,FASTCALL
|
|
|
|
if DBG
|
|
extrn _KdDebuggerEnabled:BYTE
|
|
EXTRNP _DbgBreakPoint,0
|
|
EXTRNP _KdPollBreakIn,0
|
|
extrn _DbgPrint:near
|
|
extrn _MsgDpcTrashedEsp:BYTE
|
|
extrn _MsgDpcTimeout:BYTE
|
|
extrn _KiDPCTimeout:DWORD
|
|
endif
|
|
|
|
|
|
_TEXT$00 SEGMENT PARA PUBLIC 'CODE'
|
|
ASSUME DS:FLAT, ES:FLAT, SS:NOTHING, FS:NOTHING, GS:NOTHING
|
|
|
|
cPublicFastCall KiRDTSC, 1
|
|
rdtsc ; read the timestamp counter
|
|
mov [ecx], eax ; return the low 32 bits
|
|
mov [ecx+4], edx ; return the high 32 bits
|
|
fstRET KiRDTSC
|
|
fstENDP KiRDTSC
|
|
|
|
page ,132
|
|
subttl "Unlock Dispatcher Database"
|
|
;++
|
|
;
|
|
; VOID
|
|
; KiUnlockDispatcherDatabase (
|
|
; IN KIRQL OldIrql
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine is entered at IRQL DISPATCH_LEVEL with the dispatcher
|
|
; database locked. Its function is to either unlock the dispatcher
|
|
; database and return or initiate a context switch if another thread
|
|
; has been selected for execution.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (TOS) Return address
|
|
;
|
|
; (ecx) OldIrql - Supplies the IRQL when the dispatcher database
|
|
; lock was acquired.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall KiUnlockDispatcherDatabase, 1
|
|
|
|
;
|
|
; Check if a new thread is scheduled for execution.
|
|
;
|
|
|
|
cmp PCR[PcPrcbData+PbNextThread], 0 ; check if next thread
|
|
jne short Kiu20 ; if ne, new thread scheduled
|
|
|
|
;
|
|
; Release dispatcher database lock, lower IRQL to its previous level,
|
|
; and return.
|
|
;
|
|
|
|
Kiu00: ;
|
|
|
|
ifndef NT_UP
|
|
|
|
mov eax, PCR[PcPrcb] ; get address of PRCB
|
|
push ecx ; save IRQL for lower IRQL
|
|
lea ecx, [eax]+PbLockQueue+(8*LockQueueDispatcherLock)
|
|
fstCall KeReleaseQueuedSpinLockFromDpcLevel
|
|
pop ecx ; get OldIrql
|
|
|
|
endif
|
|
|
|
;
|
|
; N.B. This exit jumps directly to the lower IRQL routine which has a
|
|
; compatible fastcall interface.
|
|
;
|
|
|
|
jmp dword ptr [__imp_@KfLowerIrql@4] ; lower IRQL to previous level
|
|
|
|
;
|
|
; A new thread has been selected to run on the current processor, but
|
|
; the new IRQL is not below dispatch level. If the current processor is
|
|
; not executing a DPC, then request a dispatch interrupt on the current
|
|
; processor.
|
|
;
|
|
|
|
Kiu10: cmp dword ptr PCR[PcPrcbData.PbDpcRoutineActive],0 ; check if DPC routine active
|
|
jne short Kiu00 ; if ne, DPC routine is active
|
|
|
|
push ecx ; save new IRQL
|
|
|
|
ifndef NT_UP
|
|
|
|
mov eax, PCR[PcPrcb] ; get address of PRCB
|
|
lea ecx, [eax]+PbLockQueue+(8*LockQueueDispatcherLock)
|
|
fstCall KeReleaseQueuedSpinLockFromDpcLevel
|
|
|
|
endif
|
|
|
|
mov cl, DISPATCH_LEVEL ; request dispatch interrupt
|
|
fstCall HalRequestSoftwareInterrupt ;
|
|
pop ecx ; restore new IRQL
|
|
|
|
;
|
|
; N.B. This exit jumps directly to the lower IRQL routine which has a
|
|
; compatible fastcall interface.
|
|
;
|
|
|
|
jmp dword ptr [__imp_@KfLowerIrql@4] ; lower IRQL to previous level
|
|
|
|
;
|
|
; Check if the previous IRQL is less than dispatch level.
|
|
;
|
|
|
|
Kiu20: cmp cl, DISPATCH_LEVEL ; check if IRQL below dispatch level
|
|
jge short Kiu10 ; if ge, not below dispatch level
|
|
|
|
;
|
|
; There is a new thread scheduled for execution and the previous IRQL is
|
|
; less than dispatch level. Context switch to the new thread immediately.
|
|
;
|
|
;
|
|
; N.B. The following registers MUST be saved such that ebp is saved last.
|
|
; This is done so the debugger can find the saved ebp for a thread
|
|
; that is not currently in the running state.
|
|
;
|
|
|
|
.fpo (0, 0, 0, 4, 1, 0)
|
|
sub esp, 4*4
|
|
mov [esp+12], ebx ; save registers
|
|
mov [esp+8], esi ;
|
|
mov [esp+4], edi ;
|
|
mov [esp+0], ebp ;
|
|
mov ebx, PCR[PcSelfPcr] ; get address of PCR
|
|
mov esi, [ebx].PcPrcbData.PbNextThread ; get next thread address
|
|
mov edi, [ebx].PcPrcbData.PbCurrentThread ; get current thread address
|
|
mov dword ptr [ebx].PcPrcbData.PbNextThread, 0 ; clear next thread address
|
|
mov [ebx].PcPrcbData.PbCurrentThread, esi ; set current thread address
|
|
mov [edi].ThWaitIrql, cl ; save previous IRQL
|
|
mov ecx, edi ; set address of current thread
|
|
mov byte ptr [edi].ThIdleSwapBlock, 1
|
|
fstCall KiReadyThread ; reready thread for execution
|
|
mov cl, [edi].ThWaitIrql ; set APC interrupt bypass disable
|
|
CAPSTART <@KiUnlockDispatcherDatabase@4,SwapContext>
|
|
call SwapContext ; swap context
|
|
CAPEND <@KiUnlockDispatcherDatabase@4>
|
|
or al, al ; check if kernel APC pending
|
|
mov cl, [esi].ThWaitIrql ; get original wait IRQL
|
|
jnz short Kiu50 ; if nz, kernel APC pending
|
|
|
|
Kiu30: mov ebp, [esp+0] ; restore registers
|
|
mov edi, [esp+4] ;
|
|
mov esi, [esp+8] ;
|
|
mov ebx, [esp+12] ;
|
|
add esp, 4*4
|
|
|
|
;
|
|
; N.B. This exit jumps directly to the lower IRQL routine which has a
|
|
; compatible fastcall interface.
|
|
;
|
|
|
|
jmp dword ptr [__imp_@KfLowerIrql@4] ; lower IRQL to previous level
|
|
|
|
Kiu50: mov cl, APC_LEVEL ; lower IRQL to APC level
|
|
fstCall KfLowerIrql ;
|
|
xor eax, eax ; set previous mode to kernel
|
|
CAPSTART <@KiUnlockDispatcherDatabase@4,_KiDeliverApc@12>
|
|
stdCall _KiDeliverApc, <eax, eax, eax> ; deliver kernel mode APC
|
|
CAPEND <@KiUnlockDispatcherDatabase@4>
|
|
xor ecx, ecx ; set original wait IRQL
|
|
jmp short Kiu30
|
|
|
|
fstENDP KiUnlockDispatcherDatabase
|
|
|
|
page ,132
|
|
subttl "Swap Context"
|
|
;++
|
|
;
|
|
; BOOLEAN
|
|
; KiSwapContext (
|
|
; IN PKTHREAD Thread
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function is a small wrapper, callable from C code, that marshalls
|
|
; arguments and calls the actual swap context routine.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Thread (ecx) - Supplies the address of the new thread.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; If a kernel APC is pending, then a value of TRUE is returned. Otherwise,
|
|
; a value of FALSE is returned.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall KiSwapContext, 1
|
|
.fpo (0, 0, 0, 4, 1, 0)
|
|
|
|
;
|
|
; N.B. The following registers MUST be saved such that ebp is saved last.
|
|
; This is done so the debugger can find the saved ebp for a thread
|
|
; that is not currently in the running state.
|
|
;
|
|
|
|
sub esp, 4*4
|
|
mov [esp+12], ebx ; save registers
|
|
mov [esp+8], esi ;
|
|
mov [esp+4], edi ;
|
|
mov [esp+0], ebp ;
|
|
mov ebx, PCR[PcSelfPcr] ; set address of PCR
|
|
mov esi, ecx ; set next thread address
|
|
mov edi, [ebx].PcPrcbData.PbCurrentThread ; get current thread address
|
|
mov [ebx].PcPrcbData.PbCurrentThread, esi ; set current thread address
|
|
mov cl, [edi].ThWaitirql ; set APC interrupt bypass disable
|
|
call SwapContext ; swap context
|
|
mov ebp, [esp+0] ; restore registers
|
|
mov edi, [esp+4] ;
|
|
mov esi, [esp+8] ;
|
|
mov ebx, [esp+12] ;
|
|
add esp, 4*4 ;
|
|
fstRET KiSwapContext ;
|
|
|
|
fstENDP KiSwapContext
|
|
|
|
page ,132
|
|
subttl "Dispatch Interrupt"
|
|
;++
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine is entered as the result of a software interrupt generated
|
|
; at DISPATCH_LEVEL. Its function is to process the Deferred Procedure Call
|
|
; (DPC) list, and then perform a context switch if a new thread has been
|
|
; selected for execution on the processor.
|
|
;
|
|
; This routine is entered at IRQL DISPATCH_LEVEL with the dispatcher
|
|
; database unlocked. When a return to the caller finally occurs, the
|
|
; IRQL remains at DISPATCH_LEVEL, and the dispatcher database is still
|
|
; unlocked.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; None
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicProc _KiDispatchInterrupt ,0
|
|
cPublicFpo 0, 0
|
|
|
|
mov ebx, PCR[PcSelfPcr] ; get address of PCR
|
|
kdi00: lea eax, [ebx].PcPrcbData.PbDpcListHead ; get DPC listhead address
|
|
|
|
;
|
|
; Disable interrupts and check if there is any work in the DPC list
|
|
; of the current processor.
|
|
;
|
|
|
|
kdi10: cli ; disable interrupts
|
|
cmp eax, [eax].LsFlink ; check if DPC List is empty
|
|
je short kdi40 ; if eq, list is empty
|
|
push ebp ; save register
|
|
|
|
;
|
|
; Exceptions occuring in DPCs are unrelated to any exception handlers
|
|
; in the interrupted thread. Terminate the exception list.
|
|
;
|
|
|
|
push [ebx].PcExceptionList
|
|
mov [ebx].PcExceptionList, EXCEPTION_CHAIN_END
|
|
|
|
;
|
|
; Switch to the DPC stack for this processor.
|
|
;
|
|
|
|
mov edx, esp
|
|
mov esp, [ebx].PcPrcbData.PbDpcStack
|
|
push edx
|
|
|
|
.fpo (0, 0, 0, 1, 1, 0)
|
|
|
|
mov ebp, eax ; set address of DPC listhead
|
|
CAPSTART <_KiDispatchInterrupt,KiRetireDpcList>
|
|
call KiRetireDpcList ; process the current DPC list
|
|
CAPEND <_KiDispatchInterrupt>
|
|
|
|
;
|
|
; Switch back to the current thread stack, restore the exception list
|
|
; and saved EBP.
|
|
;
|
|
|
|
pop esp
|
|
pop [ebx].PcExceptionList
|
|
pop ebp
|
|
.fpo (0, 0, 0, 0, 0, 0)
|
|
|
|
;
|
|
; Check to determine if quantum end is requested.
|
|
;
|
|
; N.B. If a new thread is selected as a result of processing the quantum
|
|
; end request, then the new thread is returned with the dispatcher
|
|
; database locked. Otherwise, NULL is returned with the dispatcher
|
|
; database unlocked.
|
|
;
|
|
|
|
kdi40: sti ; enable interrupts
|
|
cmp dword ptr [ebx].PcPrcbData.PbQuantumEnd, 0 ; quantum end requested
|
|
jne kdi90 ; if neq, quantum end request
|
|
|
|
;
|
|
; Check to determine if a new thread has been selected for execution on this
|
|
; processor.
|
|
;
|
|
|
|
cmp dword ptr [ebx].PcPrcbData.PbNextThread, 0 ; check addr of next thread object
|
|
je kdi70 ; if eq, then no new thread
|
|
|
|
;
|
|
; Disable interrupts and attempt to acquire the dispatcher database lock.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
cli
|
|
cmp dword ptr _KiDispatcherLock, 0
|
|
jnz short kdi80
|
|
lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueDispatcherLock)
|
|
fstCall KeTryToAcquireQueuedSpinLockAtRaisedIrql
|
|
jz short kdi80 ; jif not acquired
|
|
mov ecx, SYNCH_LEVEL ; raise IRQL to synchronization level
|
|
fstCall KfRaiseIrql ;
|
|
sti ; enable interrupts
|
|
|
|
endif
|
|
|
|
mov eax, [ebx].PcPrcbData.PbNextThread ; get next thread address
|
|
|
|
;
|
|
; N.B. The following registers MUST be saved such that ebp is saved last.
|
|
; This is done so the debugger can find the saved ebp for a thread
|
|
; that is not currently in the running state.
|
|
;
|
|
|
|
.fpo (0, 0, 0, 3, 1, 0)
|
|
|
|
kdi60: sub esp, 3*4
|
|
mov [esp+8], esi ; save registers
|
|
mov [esp+4], edi ;
|
|
mov [esp+0], ebp ;
|
|
mov esi, eax ; set next thread address
|
|
mov edi, [ebx].PcPrcbData.PbCurrentThread ; get current thread address
|
|
mov dword ptr [ebx].PcPrcbData.PbNextThread, 0 ; clear next thread address
|
|
mov [ebx].PcPrcbData.PbCurrentThread, esi ; set current thread address
|
|
mov ecx, edi ; set address of current thread
|
|
mov byte ptr [edi].ThIdleSwapBlock, 1
|
|
fstCall KiReadyThread ; reready thread for execution
|
|
CAPSTART <_KiDispatchInterrupt,SwapContext>
|
|
mov cl, 1 ; set APC interrupt bypass disable
|
|
call SwapContext ; swap context
|
|
CAPEND <_KiDispatchInterrupt>
|
|
mov ebp, [esp+0] ; restore registers
|
|
mov edi, [esp+4] ;
|
|
mov esi, [esp+8] ;
|
|
add esp, 3*4
|
|
kdi70: stdRET _KiDispatchInterrupt ; return
|
|
|
|
;
|
|
; Enable interrupts and check DPC queue.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
kdi80: sti ; enable interrupts
|
|
YIELD ; rest
|
|
jmp kdi00 ;
|
|
|
|
endif
|
|
|
|
;
|
|
; Process quantum end event.
|
|
;
|
|
; N.B. If the quantum end code returns a NULL value, then no next thread
|
|
; has been selected for execution. Otherwise, a next thread has been
|
|
; selected and the dispatcher databased is locked.
|
|
;
|
|
|
|
kdi90: mov dword ptr [ebx].PcPrcbData.PbQuantumEnd, 0 ; clear quantum end indicator
|
|
CAPSTART <_KiDispatchInterrupt,_KiQuantumEnd@0>
|
|
stdCall _KiQuantumEnd ; process quantum end
|
|
CAPEND <_KiDispatchInterrupt>
|
|
or eax, eax ; check if new thread selected
|
|
jne kdi60 ; if ne, new thread selected
|
|
stdRET _KiDispatchInterrupt ; return
|
|
|
|
stdENDP _KiDispatchInterrupt
|
|
|
|
page ,132
|
|
subttl "Swap Context to Next Thread"
|
|
;++
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine is called to swap context from one thread to the next.
|
|
; It swaps context, flushes the data, instruction, and translation
|
|
; buffer caches, restores nonvolatile integer registers, and returns
|
|
; to its caller.
|
|
;
|
|
; N.B. It is assumed that the caller (only callers are within this
|
|
; module) saved the nonvolatile registers, ebx, esi, edi, and
|
|
; ebp. This enables the caller to have more registers available.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; cl - APC interrupt bypass disable (zero enable, nonzero disable).
|
|
; edi - Address of previous thread.
|
|
; esi - Address of next thread.
|
|
; ebx - Address of PCR.
|
|
;
|
|
; Return value:
|
|
;
|
|
; al - Kernel APC pending.
|
|
; ebx - Address of PCR.
|
|
; esi - Address of current thread object.
|
|
;
|
|
;--
|
|
|
|
;
|
|
; NOTE: The ES: override on the move to ThState is part of the
|
|
; lazy-segment load system. It assures that ES has a valid
|
|
; selector in it, thus preventing us from propagating a bad
|
|
; ES accross a context switch.
|
|
;
|
|
; Note that if segments, other than the standard flat segments,
|
|
; with limits above 2 gig exist, neither this nor the rest of
|
|
; lazy segment loads are reliable.
|
|
;
|
|
; Note that ThState must be set before the dispatcher lock is released
|
|
; to prevent KiSetPriorityThread from seeing a stale value.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
public _ScPatchFxb
|
|
public _ScPatchFxe
|
|
|
|
endif
|
|
|
|
align 16
|
|
public SwapContext
|
|
|
|
SwapContext proc
|
|
|
|
;
|
|
; Save the APC disable flag and set new thread state to running.
|
|
;
|
|
|
|
or cl, cl ; set zf in flags
|
|
mov byte ptr es:[esi]+ThState, Running ; set thread state to running
|
|
pushfd
|
|
cPublicFpo 0, 1
|
|
|
|
;
|
|
; Acquire the context swap lock so the address space of the old process
|
|
; cannot be deleted and then release the dispatcher database lock.
|
|
;
|
|
; N.B. This lock is used to protect the address space until the context
|
|
; switch has sufficiently progressed to the point where the address
|
|
; space is no longer needed. This lock is also acquired by the reaper
|
|
; thread before it finishes thread termination.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueContextSwapLock)
|
|
fstCall KeAcquireQueuedSpinLockAtDpcLevel ; Acquire ContextSwap lock
|
|
|
|
lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueDispatcherLock)
|
|
fstCall KeReleaseQueuedSpinLockFromDpcLevel ; release Dispatcher Lock
|
|
|
|
endif
|
|
|
|
;
|
|
; Save the APC disable flag and the exception listhead.
|
|
; (also, check for DPC running which is illegal right now).
|
|
;
|
|
|
|
SwapContextFromIdle:
|
|
|
|
mov ecx, [ebx]+PcExceptionList ; save exception list
|
|
cmp [ebx]+PcPrcbData+PbDpcRoutineActive, 0
|
|
push ecx
|
|
cPublicFpo 0, 2
|
|
jne sc91 ; bugcheck if DPC active.
|
|
|
|
;
|
|
; Check for context swap logging
|
|
;
|
|
|
|
|
|
|
|
cmp _PPerfGlobalGroupMask, 0 ; Is the global pointer != null?
|
|
jne sc92 ; If not, then check if we are enabled
|
|
sc03:
|
|
|
|
ifndef NT_UP
|
|
|
|
if DBG
|
|
|
|
mov cl, [esi]+ThNextProcessor ; get current processor number
|
|
cmp cl, [ebx]+PcPrcbData+PbNumber ; same as running processor?
|
|
jne sc_error2 ; if ne, processor number mismatch
|
|
|
|
endif
|
|
endif
|
|
|
|
;
|
|
; Accumulate the total time spent in a thread.
|
|
;
|
|
|
|
ifdef PERF_DATA
|
|
|
|
test _KeFeatureBits, KF_RDTSC ; feature supported?
|
|
jz short @f ; if z, feature not present
|
|
|
|
rdtsc ; read cycle counter
|
|
|
|
sub eax, [ebx].PcPrcbData.PbThreadStartCount.LiLowPart ; sub off thread
|
|
sbb edx, [ebx].PcPrcbData.PbThreadStartCount.LiHighPart ; starting time
|
|
add [edi].EtPerformanceCountLow, eax ; accumlate thread run time
|
|
adc [edi].EtPerformanceCountHigh, edx ;
|
|
add [ebx].PcPrcbData.PbThreadStartCount.LiLowPart, eax ; set new thread
|
|
adc [ebx].PcPrcbData.PbThreadStartCount.LiHighPart, edx ; starting time
|
|
@@: ;
|
|
|
|
endif
|
|
|
|
;
|
|
; On a uniprocessor system the NPX state is swapped in a lazy manner.
|
|
; If a thread whose state is not in the coprocessor attempts to perform
|
|
; a coprocessor operation, the current NPX state is swapped out (if needed),
|
|
; and the new state is swapped in durning the fault. (KiTrap07)
|
|
;
|
|
; On a multiprocessor system we still fault in the NPX state on demand, but
|
|
; we save the state when the thread switches out (assuming the NPX state
|
|
; was loaded). This is because it could be difficult to obtain the thread's
|
|
; NPX in the trap handler if it was loaded into a different processor's
|
|
; coprocessor.
|
|
;
|
|
mov ebp, cr0 ; get current CR0
|
|
mov edx, ebp
|
|
|
|
ifndef NT_UP
|
|
cmp byte ptr [edi]+ThNpxState, NPX_STATE_LOADED ; check if NPX state
|
|
je sc_save_npx_state
|
|
endif
|
|
|
|
|
|
sc05: mov cl, [esi]+ThDebugActive ; get debugger active state
|
|
mov [ebx]+PcDebugActive, cl ; set new debugger active state
|
|
|
|
;
|
|
; Switch stacks:
|
|
;
|
|
; 1. Save old esp in old thread object.
|
|
; 2. Copy stack base and stack limit into TSS AND PCR
|
|
; 3. Load esp from new thread object
|
|
;
|
|
; Keep interrupts off so we don't confuse the trap handler into thinking
|
|
; we've overrun the kernel stack.
|
|
;
|
|
|
|
cli ; disable interrupts
|
|
mov [edi]+ThKernelStack, esp ; save old kernel stack pointer
|
|
mov eax, [esi]+ThInitialStack ; get new initial stack pointer
|
|
mov ecx, [esi]+ThStackLimit ; get stack limit
|
|
sub eax, NPX_FRAME_LENGTH ; space for NPX_FRAME & NPX CR0 flags
|
|
mov [ebx]+PcStackLimit, ecx ; set new stack limit
|
|
mov [ebx]+PcInitialStack, eax ; set new stack base
|
|
|
|
.errnz (NPX_STATE_NOT_LOADED - CR0_TS - CR0_MP)
|
|
.errnz (NPX_STATE_LOADED - 0)
|
|
|
|
; (eax) = Initial Stack
|
|
; (ebx) = PCR
|
|
; (edi) = OldThread
|
|
; (esi) = NewThread
|
|
; (ebp) = Current CR0
|
|
; (edx) = Current CR0
|
|
|
|
xor ecx, ecx
|
|
mov cl, [esi]+ThNpxState ; New NPX state is (or is not) loaded
|
|
|
|
and edx, NOT (CR0_MP+CR0_EM+CR0_TS) ; clear thread settable NPX bits
|
|
or ecx, edx ; or in new thread's cr0
|
|
or ecx, [eax]+FpCr0NpxState ; merge new thread settable state
|
|
cmp ebp, ecx ; check if old and new CR0 match
|
|
jne sc_reload_cr0 ; if ne, no change in CR0
|
|
|
|
;
|
|
; N.B. It is important that the following adjustment NOT be applied to
|
|
; the initial stack value in the PCR. If it is, it will cause the
|
|
; location in memory that the processor pushes the V86 mode segment
|
|
; registers and the first 4 ULONGs in the FLOATING_SAVE_AREA to
|
|
; occupy the same memory locations, which could result in either
|
|
; trashed segment registers in V86 mode, or a trashed NPX state.
|
|
;
|
|
; Adjust ESP0 so that V86 mode threads and 32 bit threads can share
|
|
; a trapframe structure, and the NPX save area will be accessible
|
|
; in the same manner on all threads
|
|
;
|
|
; This test will check the user mode flags. On threads with no user
|
|
; mode context, the value of esp0 does not matter (we will never run
|
|
; in user mode without a usermode context, and if we don't run in user
|
|
; mode the processor will never use the esp0 value.
|
|
;
|
|
|
|
align 4
|
|
sc06: test dword ptr [eax] - KTRAP_FRAME_LENGTH + TsEFlags, EFLAGS_V86_MASK
|
|
jnz short sc07 ; if nz, V86 frame, no adjustment
|
|
sub eax, TsV86Gs - TsHardwareSegSs ; bias for missing fields
|
|
sc07: mov ecx, [ebx]+PcTss ;
|
|
mov [ecx]+TssEsp0, eax ;
|
|
mov esp, [esi]+ThKernelStack ; set new stack pointer
|
|
mov eax, [esi]+ThTeb ; get user TEB address
|
|
mov [ebx]+PcTeb, eax ; set user TEB address
|
|
sti ; enable interrupts
|
|
|
|
;
|
|
; If the new process is not the same as the old process, then switch the
|
|
; address space to the new process.
|
|
;
|
|
|
|
mov eax, [edi].ThApcState.AsProcess ; get old process address
|
|
cmp eax, [esi].ThApcState.AsProcess ; check if process match
|
|
|
|
;
|
|
; The old thread is no longer on the processor. It is no longer
|
|
; necessary to protect context swap against an idle processor picking
|
|
; up that thread before it has been completely removed from this
|
|
; processor.
|
|
;
|
|
|
|
mov byte ptr [edi].ThIdleSwapBlock, 0
|
|
|
|
jz short sc22 ; if z old process same as new
|
|
mov edi, [esi].ThApcState.AsProcess ; get new process address
|
|
;
|
|
; NOTE: Keep KiSwapProcess (below) in sync with this code!
|
|
;
|
|
; Update the processor set masks.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
mov ecx, [ebx]+PcSetMember ; get processor set member
|
|
xor [eax]+PrActiveProcessors, ecx ; clear bit in old processor set
|
|
xor [edi]+PrActiveProcessors, ecx ; set bit in new processor set
|
|
|
|
if DBG
|
|
test [eax]+PrActiveProcessors, ecx ; test if bit clear in old set
|
|
jnz sc_error4 ; if nz, bit not clear in old set
|
|
test [edi]+PrActiveProcessors, ecx ; test if bit set in new set
|
|
jz sc_error5 ; if z, bit not set in new set
|
|
|
|
endif
|
|
endif
|
|
|
|
;
|
|
; LDT switch
|
|
;
|
|
|
|
test word ptr [edi]+PrLdtDescriptor, 0FFFFH
|
|
jnz short sc_load_ldt ; if nz, LDT limit
|
|
xor eax, eax ; set LDT NULL
|
|
sc21: lldt ax
|
|
|
|
;
|
|
; All system structures have been updated, release the context swap
|
|
; lock. This thread is still at raised IRQL so it cannot be context
|
|
; switched, remaining setup can be done without the lock held.
|
|
;
|
|
|
|
ifndef NT_UP
|
|
|
|
lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueContextSwapLock)
|
|
fstCall KeReleaseQueuedSpinLockFromDpcLevel
|
|
|
|
endif
|
|
|
|
;
|
|
; New CR3, flush tb, sync tss, set IOPM
|
|
; CS, SS, DS, ES all have flat (GDT) selectors in them.
|
|
; FS has the pcr selector.
|
|
; Therefore, GS is only selector we need to flush. We null it out,
|
|
; it will be reloaded from a stack frame somewhere above us.
|
|
; Note: this load of GS before CR3 works around P6 step B0 errata 11
|
|
;
|
|
|
|
xor eax, eax
|
|
mov gs, eax ; (16bit gs = 32bit eax, truncates
|
|
; saves 1 byte, is faster).
|
|
mov eax, [edi]+PrDirectoryTableBase ; get new directory base
|
|
mov ebp, [ebx]+PcTss ; get new TSS
|
|
mov ecx, [edi]+PrIopmOffset ; get IOPM offset
|
|
mov [ebp]+TssCR3, eax ; make TSS be in sync with hardware
|
|
mov cr3, eax ; flush TLB and set new directory base
|
|
mov [ebp]+TssIoMapBase, cx ;
|
|
jmp short sc23
|
|
|
|
;
|
|
; Release the context swap lock.
|
|
;
|
|
|
|
align 4
|
|
sc22: ;
|
|
|
|
ifndef NT_UP
|
|
|
|
lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueContextSwapLock)
|
|
fstCall KeReleaseQueuedSpinLockFromDpcLevel
|
|
|
|
endif
|
|
|
|
;
|
|
; Edit the TEB descriptor to point to the TEB
|
|
;
|
|
|
|
sc23:
|
|
mov eax, [ebx]+PcTeb
|
|
mov ecx, [ebx]+PcGdt ;
|
|
mov [ecx]+(KGDT_R3_TEB+KgdtBaseLow), ax ;
|
|
shr eax, 16 ;
|
|
mov [ecx]+(KGDT_R3_TEB+KgdtBaseMid), al ;
|
|
mov [ecx]+(KGDT_R3_TEB+KgdtBaseHi), ah
|
|
|
|
;
|
|
; Update context switch counters.
|
|
;
|
|
|
|
inc dword ptr [esi]+ThContextSwitches ; thread count
|
|
inc dword ptr [ebx]+PcPrcbData+PbContextSwitches ; processor count
|
|
pop ecx ; restore exception list
|
|
mov [ebx].PcExceptionList, ecx ;
|
|
|
|
;
|
|
; If the new thread has a kernel mode APC pending, then request an APC
|
|
; interrupt.
|
|
;
|
|
|
|
cmp byte ptr [esi].ThApcState.AsKernelApcPending, 0 ; APC pending?
|
|
jne short sc80 ; if ne, kernel APC pending
|
|
popfd ; restore flags
|
|
xor eax, eax ; clear kernel APC pending
|
|
ret ; return
|
|
|
|
;
|
|
; The new thread has an APC interrupt pending. If APC interrupt bypass is
|
|
; enable, then return kernel APC pending. Otherwise, request a software
|
|
; interrupt at APC_LEVEL and return no kernel APC pending.
|
|
;
|
|
|
|
sc80: popfd ; restore flags
|
|
jnz short sc90 ; if nz, APC interupt bypass disabled
|
|
mov al, 1 ; set kernel APC pending
|
|
ret ;
|
|
|
|
sc90: mov cl, APC_LEVEL ; request software interrupt level
|
|
fstCall HalRequestSoftwareInterrupt ;
|
|
xor eax, eax ; clear kernel APC pending
|
|
ret ;
|
|
|
|
;
|
|
; Set for new LDT value
|
|
;
|
|
|
|
sc_load_ldt:
|
|
mov ebp, [ebx]+PcGdt ;
|
|
mov eax, [edi+PrLdtDescriptor] ;
|
|
mov [ebp+KGDT_LDT], eax ;
|
|
mov eax, [edi+PrLdtDescriptor+4] ;
|
|
mov [ebp+KGDT_LDT+4], eax ;
|
|
mov eax, KGDT_LDT ;
|
|
|
|
;
|
|
; Set up int 21 descriptor of IDT. If the process does not have an Ldt, it
|
|
; should never make any int 21 calls. If it does, an exception is generated. If
|
|
; the process has an Ldt, we need to update int21 entry of LDT for the process.
|
|
; Note the Int21Descriptor of the process may simply indicate an invalid
|
|
; entry. In which case, the int 21 will be trapped to the kernel.
|
|
;
|
|
|
|
mov ebp, [ebx]+PcIdt ;
|
|
mov ecx, [edi+PrInt21Descriptor] ;
|
|
mov [ebp+21h*8], ecx ;
|
|
mov ecx, [edi+PrInt21Descriptor+4] ;
|
|
mov [ebp+21h*8+4], ecx ;
|
|
jmp sc21
|
|
|
|
;
|
|
; Cr0 has changed (ie, floating point processor present), load the new value.
|
|
;
|
|
|
|
sc_reload_cr0:
|
|
if DBG
|
|
|
|
test byte ptr [esi]+ThNpxState, NOT (CR0_TS+CR0_MP)
|
|
jnz sc_error ;
|
|
test dword ptr [eax]+FpCr0NpxState, NOT (CR0_PE+CR0_MP+CR0_EM+CR0_TS)
|
|
jnz sc_error3 ;
|
|
|
|
endif
|
|
mov cr0,ecx ; set new CR0 NPX state
|
|
jmp sc06
|
|
|
|
|
|
ifndef NT_UP
|
|
|
|
|
|
; Save coprocessor's current context. FpCr0NpxState is the current thread's
|
|
; CR0 state. The following bits are valid: CR0_MP, CR0_EM, CR0_TS. MVDMs
|
|
; may set and clear MP & EM as they please and the settings will be reloaded
|
|
; on a context switch (but they will not be saved from CR0 to Cr0NpxState).
|
|
; The kernel sets and clears TS as required.
|
|
;
|
|
; (ebp) = Current CR0
|
|
; (edx) = Current CR0
|
|
|
|
sc_save_npx_state:
|
|
and edx, NOT (CR0_MP+CR0_EM+CR0_TS) ; we need access to the NPX state
|
|
|
|
mov ecx,[ebx]+PcInitialStack ; get NPX save save area address
|
|
|
|
cmp ebp, edx ; Does CR0 need reloading?
|
|
je short sc_npx10
|
|
|
|
mov cr0, edx ; set new cr0
|
|
mov ebp, edx ; (ebp) = (edx) = current cr0 state
|
|
|
|
sc_npx10:
|
|
;
|
|
; The fwait following the fnsave is to make sure that the fnsave has stored the
|
|
; data into the save area before this coprocessor state could possibly be
|
|
; context switched in and used on a different (co)processor. I've added the
|
|
; clocks from when the dispatcher lock is released and don't believe it's a
|
|
; possibility. I've also timed the impact this fwait seems to have on a 486
|
|
; when performing lots of numeric calculations. It appears as if there is
|
|
; nothing to wait for after the fnsave (although the 486 manual says there is)
|
|
; and therefore the calculation time far outweighed the 3clk fwait and it
|
|
; didn't make a noticable difference.
|
|
;
|
|
|
|
;
|
|
; If FXSR feature is NOT present on the processor, the fxsave instruction is
|
|
; patched at boot time to start using fnsave instead
|
|
;
|
|
|
|
_ScPatchFxb:
|
|
; fxsave [ecx] ; save NPX state
|
|
db 0FH, 0AEH, 01
|
|
_ScPatchFxe:
|
|
|
|
mov byte ptr [edi]+ThNpxState, NPX_STATE_NOT_LOADED ; set no NPX state
|
|
mov dword ptr [ebx].PcPrcbData+PbNpxThread, 0 ; clear npx owner
|
|
jmp sc05
|
|
endif
|
|
|
|
|
|
;
|
|
; This code is out of line to optimize the normal case
|
|
; (which is expected to be the case where tracing is off)
|
|
; First we grab the pointer to our flags struct and place it in eax.
|
|
; To take advantage of spare cycles while the load is occuring, we
|
|
; prepare for our upcoming funciton call by copying parameters between regs.
|
|
; Next we dereference that pointer plus the offset of the flag we need
|
|
; to check and bitwise AND that value with our flag. If the result is
|
|
; nonzero, then we make the function call to trace this context swap.
|
|
;
|
|
; NOTE: The flags struct is a static global.
|
|
;
|
|
sc92:
|
|
mov eax, _PPerfGlobalGroupMask ; Load the ptr into eax
|
|
cmp eax, 0 ; catch race here
|
|
jz sc03
|
|
mov edx, esi ; pass the new ETHREAD object
|
|
mov ecx, edi ; pass the old ETHREAD object
|
|
test dword ptr [eax+PERF_CONTEXTSWAP_OFFSET], PERF_CONTEXTSWAP_FLAG
|
|
jz sc03 ; return if our flag is not set
|
|
|
|
fstCall WmiTraceContextSwap ; call the Wmi context swap trace
|
|
jmp sc03
|
|
|
|
.fpo (2, 0, 0, 0, 0, 0)
|
|
sc91: stdCall _KeBugCheck <ATTEMPTED_SWITCH_FROM_DPC>
|
|
ret ; return
|
|
|
|
if DBG
|
|
sc_error5: int 3
|
|
sc_error4: int 3
|
|
sc_error3: int 3
|
|
sc_error2: int 3
|
|
sc_error: int 3
|
|
endif
|
|
|
|
SwapContext endp
|
|
|
|
page , 132
|
|
subttl "Interlocked Swap PTE"
|
|
|
|
;++
|
|
;
|
|
; HARDWARE_PTE
|
|
; KeInterlockedSwapPte (
|
|
; IN PKI_INTERNAL_PTE PtePointer,
|
|
; IN PKI_INTERNAL_PTE NewPteContents
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function performs an interlocked swap of a PTE. This is only needed
|
|
; for the PAE architecture where the PTE width is larger than the register
|
|
; width.
|
|
;
|
|
; Both PTEs must be valid or a careful write would have been done instead.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; PtePointer - Address of Pte to update with new value.
|
|
;
|
|
; NewPteContents - Pointer to the new value to put in the Pte. Will simply
|
|
; be assigned to *PtePointer, in a fashion correct for the hardware.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; Returns the contents of the PtePointer before the new value
|
|
; is stored.
|
|
;
|
|
;--
|
|
|
|
cPublicProc _KeInterlockedSwapPte ,2
|
|
|
|
push ebx
|
|
push esi
|
|
|
|
mov ebx, [esp] + 16 ; ebx = NewPteContents
|
|
mov esi, [esp] + 12 ; esi = PtePointer
|
|
|
|
mov ecx, [ebx] + 4
|
|
mov ebx, [ebx] ; ecx:ebx = source pte contents
|
|
|
|
mov edx, [esi] + 4
|
|
mov eax, [esi] ; edx:eax = target pte contents
|
|
|
|
swapagain:
|
|
|
|
;
|
|
; cmpxchg loads edx:eax with the new contents of the target quadword
|
|
; in the event of failure
|
|
;
|
|
|
|
ifndef NT_UP
|
|
lock cmpxchg8b qword ptr [esi] ; compare and exchange
|
|
else
|
|
cmpxchg8b qword ptr [esi] ; compare and exchange
|
|
endif
|
|
|
|
jnz short swapagain ; if z clear, exchange failed
|
|
|
|
pop esi
|
|
pop ebx
|
|
|
|
stdRET _KeInterlockedSwapPte
|
|
stdENDP _KeInterlockedSwapPte
|
|
|
|
page , 132
|
|
subttl "Flush EntireTranslation Buffer"
|
|
;++
|
|
;
|
|
; VOID
|
|
; KeFlushCurrentTb (
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function flushes the entire translation buffer (TB) on the current
|
|
; processor and also flushes the data cache if an entry in the translation
|
|
; buffer has become invalid.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
cPublicProc _KeFlushCurrentTb ,0
|
|
|
|
ktb00: mov eax, cr3 ; (eax) = directory table base
|
|
mov cr3, eax ; flush TLB
|
|
stdRET _KeFlushCurrentTb
|
|
|
|
ktb_gb: mov eax, cr4 ; *** see Ki386EnableGlobalPage ***
|
|
and eax, not CR4_PGE ; This FlushCurrentTb version gets copied into
|
|
mov cr4, eax ; ktb00 at initialization time if needed.
|
|
or eax, CR4_PGE
|
|
mov cr4, eax
|
|
ktb_eb: stdRET _KeFlushCurrentTb
|
|
|
|
stdENDP _KeFlushCurrentTb
|
|
;;
|
|
;; moved KiFlushDcache below KeFlushCurrentTb for BBT purposes. BBT
|
|
;; needs an end label to treat KeFlushCurrentTb as data and to keep together.
|
|
;;
|
|
page , 132
|
|
subttl "Flush Data Cache"
|
|
;++
|
|
;
|
|
; VOID
|
|
; KiFlushDcache (
|
|
; )
|
|
;
|
|
; VOID
|
|
; KiFlushIcache (
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine does nothing on i386 and i486 systems. Why? Because
|
|
; (a) their caches are completely transparent, (b) they don't have
|
|
; instructions to flush their caches.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; None.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
cPublicProc _KiFlushDcache ,0
|
|
cPublicProc _KiFlushIcache ,0
|
|
|
|
stdRET _KiFlushIcache
|
|
|
|
stdENDP _KiFlushIcache
|
|
stdENDP _KiFlushDcache
|
|
|
|
|
|
_TEXT$00 ends
|
|
|
|
INIT SEGMENT DWORD PUBLIC 'CODE'
|
|
ASSUME DS:FLAT, ES:FLAT, SS:NOTHING, FS:NOTHING, GS:NOTHING
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; Ki386EnableGlobalPage (
|
|
; IN volatile PLONG Number
|
|
; )
|
|
;
|
|
; /*++
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine enables the global page PDE/PTE support in the system,
|
|
; and stalls until complete and them sets the current processor's cr4
|
|
; register to enable global page support.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number - Supplies a pointer to the count of the number of processors in
|
|
; the configuration.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;--
|
|
|
|
cPublicProc _Ki386EnableGlobalPage,1
|
|
push esi
|
|
push edi
|
|
push ebx
|
|
|
|
mov edx, [esp+16] ; pointer to Number
|
|
pushfd
|
|
cli
|
|
|
|
;
|
|
; Wait for all processors
|
|
;
|
|
lock dec dword ptr [edx] ; count down
|
|
egp10: YIELD
|
|
cmp dword ptr [edx], 0 ; wait for all processors to signal
|
|
jnz short egp10
|
|
|
|
cmp byte ptr PCR[PcNumber], 0 ; processor 0?
|
|
jne short egp20
|
|
|
|
;
|
|
; Install proper KeFlushCurrentTb function.
|
|
;
|
|
|
|
mov edi, ktb00
|
|
mov esi, ktb_gb
|
|
mov ecx, ktb_eb - ktb_gb + 1
|
|
rep movsb
|
|
|
|
mov byte ptr [ktb_eb], 0
|
|
|
|
;
|
|
; Wait for P0 to signal that proper flush TB handlers have been installed
|
|
;
|
|
egp20: cmp byte ptr [ktb_eb], 0
|
|
jnz short egp20
|
|
|
|
;
|
|
; Flush TB, and enable global page support
|
|
; (note load of CR4 is explicitly done before the load of CR3
|
|
; to work around P6 step B0 errata 11)
|
|
;
|
|
mov eax, cr4
|
|
and eax, not CR4_PGE ; should not be set, but let's be safe
|
|
mov ecx, cr3
|
|
mov cr4, eax
|
|
|
|
mov cr3, ecx ; Flush TB
|
|
|
|
or eax, CR4_PGE ; enable global TBs
|
|
mov cr4, eax
|
|
popfd
|
|
pop ebx
|
|
pop edi
|
|
pop esi
|
|
|
|
stdRET _Ki386EnableGlobalPage
|
|
stdENDP _Ki386EnableGlobalPage
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; Ki386EnableDE (
|
|
; IN volatile PLONG Number
|
|
; )
|
|
;
|
|
; /*++
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine sets DE bit in CR4 to enable IO breakpoints
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number - Supplies a pointer to the count of the number of processors in
|
|
; the configuration.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;--
|
|
|
|
cPublicProc _Ki386EnableDE,1
|
|
|
|
mov eax, cr4
|
|
or eax, CR4_DE
|
|
mov cr4, eax
|
|
|
|
stdRET _Ki386EnableDE
|
|
stdENDP _Ki386EnableDE
|
|
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; Ki386EnableFxsr (
|
|
; IN volatile PLONG Number
|
|
; )
|
|
;
|
|
; /*++
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine sets OSFXSR bit in CR4 to indicate that OS supports
|
|
; FXSAVE/FXRSTOR for use during context switches
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number - Supplies a pointer to the count of the number of processors in
|
|
; the configuration.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;--
|
|
|
|
cPublicProc _Ki386EnableFxsr,1
|
|
|
|
mov eax, cr4
|
|
or eax, CR4_FXSR
|
|
mov cr4, eax
|
|
|
|
stdRET _Ki386EnableFxsr
|
|
stdENDP _Ki386EnableFxsr
|
|
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; Ki386EnableXMMIExceptions (
|
|
; IN volatile PLONG Number
|
|
; )
|
|
;
|
|
; /*++
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine installs int 19 XMMI unmasked Numeric Exception handler
|
|
; and sets OSXMMEXCPT bit in CR4 to indicate that OS supports
|
|
; unmasked Katmai New Instruction technology exceptions.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number - Supplies a pointer to count of the number of processors in
|
|
; the configuration.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;--
|
|
|
|
cPublicProc _Ki386EnableXMMIExceptions,1
|
|
|
|
|
|
;Set up IDT for INT19
|
|
mov ecx,PCR[PcIdt] ;Get IDT address
|
|
lea eax, [ecx] + 098h ;XMMI exception is int 19
|
|
mov byte ptr [eax + 5], 08eh ;P=1,DPL=0,Type=e
|
|
mov word ptr [eax + 2], KGDT_R0_CODE ;Kernel code selector
|
|
mov edx, offset FLAT:_KiTrap13 ;Address of int 19 handler
|
|
mov ecx,edx
|
|
mov word ptr [eax],cx ;addr moves into low byte
|
|
shr ecx,16
|
|
mov word ptr [eax + 6],cx ;addr moves into high byte
|
|
;Enable XMMI exception handling
|
|
mov eax, cr4
|
|
or eax, CR4_XMMEXCPT
|
|
mov cr4, eax
|
|
|
|
stdRET _Ki386EnableXMMIExceptions
|
|
stdENDP _Ki386EnableXMMIExceptions
|
|
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; Ki386EnableCurrentLargePage (
|
|
; IN ULONG IdentityAddr,
|
|
; IN ULONG IdentityCr3
|
|
; )
|
|
;
|
|
; /*++
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine enables the large page PDE support in the processor.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; IdentityAddr - Supplies the linear address of the beginning of this
|
|
; function where (linear == physical).
|
|
;
|
|
; IdentityCr3 - Supplies a pointer to the temporary page directory and
|
|
; page tables that provide both the kernel (virtual ->physical) and
|
|
; identity (linear->physical) mappings needed for this function.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;--
|
|
|
|
public _Ki386EnableCurrentLargePageEnd
|
|
cPublicProc _Ki386EnableCurrentLargePage,2
|
|
mov ecx,[esp]+4 ; (ecx)-> IdentityAddr
|
|
mov edx,[esp]+8 ; (edx)-> IdentityCr3
|
|
pushfd ; save current IF state
|
|
cli ; disable interrupts
|
|
|
|
mov eax, cr3 ; (eax)-> original Cr3
|
|
mov cr3, edx ; load Cr3 with Identity mapping
|
|
|
|
sub ecx, offset _Ki386EnableCurrentLargePage
|
|
add ecx, offset _Ki386LargePageIdentityLabel
|
|
jmp ecx ; jump to (linear == physical)
|
|
|
|
_Ki386LargePageIdentityLabel:
|
|
mov ecx, cr0
|
|
and ecx, NOT CR0_PG ; clear PG bit to disable paging
|
|
mov cr0, ecx ; disable paging
|
|
jmp $+2
|
|
mov edx, cr4
|
|
or edx, CR4_PSE ; enable Page Size Extensions
|
|
mov cr4, edx
|
|
mov edx, offset OriginalMapping
|
|
or ecx, CR0_PG ; set PG bit to enable paging
|
|
mov cr0, ecx ; enable paging
|
|
jmp edx ; Return to original mapping.
|
|
|
|
OriginalMapping:
|
|
mov cr3, eax ; restore original Cr3
|
|
popfd ; restore interrupts to previous
|
|
|
|
stdRET _Ki386EnableCurrentLargePage
|
|
|
|
_Ki386EnableCurrentLargePageEnd:
|
|
|
|
stdENDP _Ki386EnableCurrentLargePage
|
|
|
|
INIT ends
|
|
|
|
_TEXT$00 SEGMENT PARA PUBLIC 'CODE'
|
|
ASSUME DS:FLAT, ES:FLAT, SS:NOTHING, FS:NOTHING, GS:NOTHING
|
|
|
|
page , 132
|
|
subttl "Flush Single Translation Buffer"
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; KiFlushSingleTb (
|
|
; IN BOOLEAN Invalid,
|
|
; IN PVOID Virtual
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function flushes a single TB entry.
|
|
;
|
|
; It only works on a 486 or greater.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Invalid - Supplies a boolean value that specifies the reason for
|
|
; flushing the translation buffer.
|
|
;
|
|
; Virtual - Supplies the virtual address of the single entry that is
|
|
; to be flushed from the translation buffer.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall KiFlushSingleTb ,2
|
|
|
|
;
|
|
; 486 or above code
|
|
;
|
|
invlpg [edx]
|
|
fstRET KiFlushSingleTb
|
|
|
|
fstENDP KiFlushSingleTb
|
|
|
|
page , 132
|
|
subttl "Swap Process"
|
|
;++
|
|
;
|
|
; VOID
|
|
; KiSwapProcess (
|
|
; IN PKPROCESS NewProcess,
|
|
; IN PKPROCESS OldProcess
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function swaps the address space to another process by flushing
|
|
; the data cache, the instruction cache, the translation buffer, and
|
|
; establishes a new directory table base.
|
|
;
|
|
; It also swaps in the LDT and IOPM of the new process. This is necessary
|
|
; to avoid bogus mismatches in SwapContext.
|
|
;
|
|
; NOTE: keep in sync with process switch part of SwapContext
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Process - Supplies a pointer to a control object of type process.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
cPublicProc _KiSwapProcess ,2
|
|
cPublicFpo 2, 0
|
|
|
|
ifndef NT_UP
|
|
|
|
;
|
|
; Acquire the context swap lock.
|
|
;
|
|
|
|
mov ecx, PCR[PcPrcb]
|
|
lea ecx, [ecx]+PbLockQueue+(8*LockQueueContextSwapLock)
|
|
fstCall KeAcquireQueuedSpinLockAtDpcLevel
|
|
|
|
mov edx,[esp]+4 ; (edx)-> New process
|
|
mov eax,[esp]+8 ; (eax)-> Old Process
|
|
|
|
;
|
|
; Clear the processor set member in the old process, set the processor
|
|
; member in the new process, and release the context swap lock.
|
|
;
|
|
|
|
mov ecx, PCR[PcSetMember]
|
|
xor [eax]+PrActiveProcessors,ecx ; clear bit in old processor set
|
|
xor [edx]+PrActiveProcessors,ecx ; set bit in new processor set
|
|
|
|
if DBG
|
|
|
|
test [eax]+PrActiveProcessors,ecx ; test if bit clear in old set
|
|
jnz kisp_error ; if nz, bit not clear in old set
|
|
test [edx]+PrActiveProcessors,ecx ; test if bit set in new set
|
|
jz kisp_error1 ; if z, bit not set in new set
|
|
|
|
endif
|
|
|
|
mov ecx, PCR[PcPrcb]
|
|
lea ecx, [ecx]+PbLockQueue+(8*LockQueueContextSwapLock)
|
|
fstCall KeReleaseQueuedSpinLockFromDpcLevel
|
|
|
|
endif
|
|
|
|
mov ecx,PCR[PcTss] ; (ecx)-> TSS
|
|
mov edx,[esp]+4 ; (edx)-> New process
|
|
|
|
;
|
|
; Change address space
|
|
;
|
|
|
|
xor eax,eax ; assume ldtr is to be NULL
|
|
mov gs,ax ; Clear gs. (also workarounds
|
|
; P6 step B0 errata 11)
|
|
mov eax,[edx]+PrDirectoryTableBase
|
|
mov [ecx]+TssCR3,eax ; be sure TSS in sync with processor
|
|
mov cr3,eax
|
|
|
|
;
|
|
; Change IOPM
|
|
;
|
|
|
|
mov ax,[edx]+PrIopmOffset
|
|
mov [ecx]+TssIoMapBase,ax
|
|
|
|
;
|
|
; Change LDT
|
|
;
|
|
|
|
xor eax, eax
|
|
cmp word ptr [edx]+PrLdtDescriptor,ax ; limit 0?
|
|
jz short kisp10 ; null LDT, go load NULL ldtr
|
|
|
|
;
|
|
; Edit LDT descriptor
|
|
;
|
|
|
|
mov ecx,PCR[PcGdt]
|
|
mov eax,[edx+PrLdtDescriptor]
|
|
mov [ecx+KGDT_LDT],eax
|
|
mov eax,[edx+PrLdtDescriptor+4]
|
|
mov [ecx+KGDT_LDT+4],eax
|
|
|
|
;
|
|
; Set up int 21 descriptor of IDT. If the process does not have Ldt, it
|
|
; should never make any int 21 call. If it does, an exception is generated.
|
|
; If the process has Ldt, we need to update int21 entry of LDT for the process.
|
|
; Note the Int21Descriptor of the process may simply indicate an invalid
|
|
; entry. In which case, the int 21 will be trapped to the kernel.
|
|
;
|
|
|
|
mov ecx, PCR[PcIdt]
|
|
mov eax, [edx+PrInt21Descriptor]
|
|
mov [ecx+21h*8], eax
|
|
mov eax, [edx+PrInt21Descriptor+4]
|
|
mov [ecx+21h*8+4], eax
|
|
|
|
mov eax,KGDT_LDT ;@@32-bit op to avoid prefix
|
|
|
|
;
|
|
; Load LDTR
|
|
;
|
|
|
|
kisp10: lldt ax
|
|
stdRET _KiSwapProcess
|
|
|
|
if DBG
|
|
kisp_error1: int 3
|
|
kisp_error: int 3
|
|
endif
|
|
|
|
stdENDP _KiSwapProcess
|
|
|
|
page ,132
|
|
subttl "Idle Loop"
|
|
;++
|
|
;
|
|
; VOID
|
|
; KiIdleLoop(
|
|
; VOID
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine continuously executes the idle loop and never returns.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; ebx - Address of the current processor's PCR.
|
|
;
|
|
; Return value:
|
|
;
|
|
; None - routine never returns.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall KiIdleLoop ,0
|
|
cPublicFpo 0, 0
|
|
|
|
lea ebp, [ebx].PcPrcbData.PbDpcListHead ; set DPC listhead address
|
|
|
|
if DBG
|
|
|
|
xor edi, edi ; reset poll breakin counter
|
|
|
|
endif
|
|
|
|
jmp short kid20 ; Skip HalIdleProcessor on first iteration
|
|
|
|
;
|
|
; There are no entries in the DPC list and a thread has not been selected
|
|
; for execution on this processor. Call the HAL so power managment can be
|
|
; performed.
|
|
;
|
|
; N.B. The HAL is called with interrupts disabled. The HAL will return
|
|
; with interrupts enabled.
|
|
;
|
|
; N.B. Use a call instruction instead of a push-jmp, as the call instruction
|
|
; executes faster and won't invalidate the processor's call-return stack
|
|
; cache.
|
|
;
|
|
|
|
kid10: lea ecx, [ebx].PcPrcbData.PbPowerState
|
|
call dword ptr [ecx].PpIdleFunction ; (ecx) = Arg0
|
|
|
|
;
|
|
; Give the debugger an opportunity to gain control on debug systems.
|
|
;
|
|
; N.B. On an MP system the lowest numbered idle processor is the only
|
|
; processor that polls for a breakin request.
|
|
;
|
|
|
|
kid20:
|
|
|
|
if DBG
|
|
ifndef NT_UP
|
|
|
|
mov eax, _KiIdleSummary ; get idle summary
|
|
mov ecx, [ebx].PcSetMember ; get set member
|
|
dec ecx ; compute right bit mask
|
|
and eax, ecx ; check if any lower bits set
|
|
jnz short CheckDpcList ; if nz, not lowest numbered
|
|
|
|
endif
|
|
|
|
dec edi ; decrement poll counter
|
|
jg short CheckDpcList ; if g, not time to poll
|
|
|
|
POLL_DEBUGGER ; check if break in requested
|
|
endif
|
|
|
|
kid30:
|
|
|
|
if DBG
|
|
ifndef NT_UP
|
|
|
|
mov edi, 20 * 1000 ; set breakin poll interval
|
|
|
|
else
|
|
|
|
mov edi, 100 ; UP idle loop has a HLT in it
|
|
|
|
endif
|
|
endif
|
|
|
|
CheckDpcList0:
|
|
YIELD
|
|
|
|
;
|
|
; Disable interrupts and check if there is any work in the DPC list
|
|
; of the current processor or a target processor.
|
|
;
|
|
|
|
CheckDpcList:
|
|
|
|
;
|
|
; N.B. The following code enables interrupts for a few cycles, then
|
|
; disables them again for the subsequent DPC and next thread
|
|
; checks.
|
|
;
|
|
|
|
sti ; enable interrupts
|
|
nop ;
|
|
nop ;
|
|
cli ; disable interrupts
|
|
|
|
;
|
|
; Process the deferred procedure call list for the current processor.
|
|
;
|
|
|
|
cmp ebp, [ebp].LsFlink ; check if DPC list is empty
|
|
je short CheckNextThread ; if eq, DPC list is empty
|
|
mov cl, DISPATCH_LEVEL ; set interrupt level
|
|
fstCall HalClearSoftwareInterrupt ; clear software interrupt
|
|
CAPSTART <@KiIdleLoop@0,KiRetireDpcList>
|
|
call KiRetireDpcList ; process the current DPC list
|
|
CAPEND <@KiIdleLoop@0>
|
|
|
|
if DBG
|
|
|
|
xor edi, edi ; clear breakin poll interval
|
|
|
|
endif
|
|
|
|
;
|
|
; Check if a thread has been selected to run on the current processor.
|
|
;
|
|
|
|
CheckNextThread: ;
|
|
cmp dword ptr [ebx].PcPrcbData.PbNextThread, 0 ; thread selected?
|
|
je short kid10 ; if eq, no thread selected
|
|
|
|
ifndef NT_UP
|
|
|
|
;
|
|
; A thread has been selected for execution on this processor. Acquire
|
|
; the dispatcher database lock, get the thread address again (it may have
|
|
; changed), clear the address of the next thread in the processor block,
|
|
; and call swap context to start execution of the selected thread.
|
|
;
|
|
; N.B. If the dispatcher database lock cannot be obtained immediately,
|
|
; then attempt to process another DPC rather than spinning on the
|
|
; dispatcher database lock.
|
|
; N.B. On MP systems, the dispatcher database is always locked at
|
|
; SYNCH level to ensure the lock is held for as short a period as
|
|
; possible (reduce contention). On UP systems there really is no
|
|
; lock, it is sufficient to be at DISPATCH level (which is the
|
|
; current level at this point in the code).
|
|
|
|
;
|
|
; Raise IRQL to synchronization level and enable interrupts.
|
|
;
|
|
|
|
mov ecx, SYNCH_LEVEL ; raise IRQL to synchronization level
|
|
fstCall KfRaiseIrql ;
|
|
|
|
endif
|
|
|
|
sti ; enable interrupts
|
|
|
|
ifndef NT_UP
|
|
|
|
;
|
|
; Acquire the context swap lock, this must be done with interrupts
|
|
; enabled to avoid deadlocks with processors that receive IPIs while
|
|
; holding the dispatcher lock.
|
|
;
|
|
; N.B. KeAcquireQueuedSpinLockAtDpcLevel doesn't touch IRQL, it
|
|
; is OK that we are at SYNCH level at this time.
|
|
|
|
lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueContextSwapLock)
|
|
fstCall KeAcquireQueuedSpinLockAtDpcLevel
|
|
|
|
endif
|
|
|
|
kidsw: mov esi, [ebx].PcPrcbData.PbNextThread ; get next thread address
|
|
mov edi, [ebx].PcPrcbData.PbCurrentThread ; get current thread address
|
|
ifndef NT_UP
|
|
cmp byte ptr [esi].ThIdleSwapBlock, 0
|
|
jne short kidndl
|
|
|
|
;
|
|
; If a thread had been scheduled for this processor but was removed from
|
|
; from eligibility (eg AffinitySet to not include this processor), then
|
|
; the NextThread could be the idle thread and this processor is marked
|
|
; as idle (again). In this case, the scheduler may assign another thread
|
|
; to this processor in the window between reading the NextThread field
|
|
; above, and zeroing it below.
|
|
;
|
|
; Detection of this condition is done here because this processor is not
|
|
; otherwise busy.
|
|
;
|
|
|
|
cmp esi, edi
|
|
je short kisame
|
|
|
|
endif
|
|
or ecx, 1 ; set APC disable
|
|
mov [ebx].PcPrcbData.PbCurrentThread, esi
|
|
|
|
;
|
|
; Other processors might be examining the Prcb->CurrentThread entry
|
|
; for this processor while holding the dispatcher database lock
|
|
; which is not held here. However, they always check the NextThread
|
|
; field first and if non NULL will acquire the context swap lock.
|
|
; Setting the CurrentThread field before clearing the NextThread
|
|
; field assures correct locking semantics.
|
|
;
|
|
|
|
mov byte ptr es:[esi]+ThState, Running ; set thread state running
|
|
mov dword ptr [ebx].PcPrcbData.PbNextThread, 0
|
|
|
|
CAPSTART <@KiIdleLoop@0,SwapContext>
|
|
|
|
;
|
|
; Set the stack as though code from SwapContext thru SwapContextFromIdle
|
|
; had been executed.
|
|
;
|
|
|
|
push FLAT:@f ; set return address
|
|
pushfd ; set saved flags
|
|
jmp SwapContextFromIdle
|
|
@@:
|
|
|
|
CAPEND <@KiIdleLoop@0>
|
|
|
|
ifndef NT_UP
|
|
mov ecx, DISPATCH_LEVEL ; lower IRQL to dispatch level
|
|
fstCall KfLowerIrql ;
|
|
endif
|
|
|
|
lea ebp, [ebx].PcPrcbData.PbDpcListHead ; set DPC listhead address
|
|
jmp kid30 ;
|
|
|
|
ifndef NT_UP
|
|
|
|
;
|
|
; The new thread is still on another processor and cannot be switched to
|
|
; yet. Drop the context swap lock and take another pass around the idle
|
|
; loop.
|
|
;
|
|
|
|
kidndl: lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueContextSwapLock)
|
|
@@: fstCall KeReleaseQueuedSpinLockFromDpcLevel
|
|
|
|
mov ecx, DISPATCH_LEVEL ; lower IRQL to dispatch level
|
|
fstCall KfLowerIrql ;
|
|
|
|
lea ebp, [ebx].PcPrcbData.PbDpcListHead ; set DPC listhead address
|
|
jmp kid30 ;
|
|
|
|
;
|
|
; The new thread is the Idle thread (same as old thread). This can happen
|
|
; rarely when a thread scheduled for this processor is made unable to run
|
|
; on this processor. As this processor has again been marked idle, other
|
|
; processors may unconditionally assign new threads to this processor.
|
|
;
|
|
; Acquire the dispatcher database lock to protect against this condition.
|
|
;
|
|
|
|
kisame: lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueContextSwapLock)
|
|
fstCall KeReleaseQueuedSpinLockFromDpcLevel
|
|
|
|
lea ecx, [ebx]+PcPrcbData+PbLockQueue+(8*LockQueueDispatcherLock)
|
|
fstCall KeAcquireQueuedSpinLockAtDpcLevel
|
|
|
|
;
|
|
; At this time, the NextThread field may have changed, if not, it is safe
|
|
; to clear it under the protection of the dispatcher lock. If it has
|
|
; changed, don't clear it.
|
|
;
|
|
|
|
cmp esi, [ebx].PcPrcbData.PbNextThread
|
|
jne short @b
|
|
mov dword ptr [ebx].PcPrcbData.PbNextThread, 0
|
|
|
|
;
|
|
; Release the dispatcher database lock and continue executing the idle
|
|
; loop. N.B. ecx still contains the address of the dispatcher database
|
|
; lock.
|
|
;
|
|
jmp short @b
|
|
|
|
endif
|
|
|
|
fstENDP KiIdleLoop
|
|
|
|
page ,132
|
|
subttl "Retire Deferred Procedure Call List"
|
|
;++
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine is called to retire the specified deferred procedure
|
|
; call list. DPC routines are called using the idle thread (current)
|
|
; stack.
|
|
;
|
|
; N.B. Interrupts must be disabled and the DPC list lock held on entry
|
|
; to this routine. Control is returned to the caller with the same
|
|
; conditions true.
|
|
;
|
|
; N.B. The registers ebx and ebp are preserved across the call.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; ebx - Address of the target processor PCR.
|
|
; ebp - Address of the target DPC listhead.
|
|
;
|
|
; Return value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
if DBG
|
|
LOCAL_OFFSET equ 4
|
|
else
|
|
LOCAL_OFFSET equ 0
|
|
endif
|
|
|
|
public KiRetireDpcList
|
|
KiRetireDpcList proc
|
|
|
|
|
|
?FpoValue = 0
|
|
|
|
ifndef NT_UP
|
|
|
|
?FpoValue = 1
|
|
push esi ; save register
|
|
lea esi, [ebx].PcPrcbData.PbDpcLock ; get DPC lock address
|
|
|
|
endif
|
|
|
|
push 0 ; Used to indicate whether DPC event logging is on or off
|
|
sub esp, 12 ; space for saved DPC address and timestamp
|
|
|
|
cmp _PPerfGlobalGroupMask, 0 ; Is event tracing on?
|
|
jne rdl70 ; go check if DPC tracing is on
|
|
rdl3:
|
|
|
|
FPOFRAME ?FpoValue,0
|
|
|
|
rdl5: mov PCR[PcPrcbData.PbDpcRoutineActive], esp ; set DPC routine active
|
|
|
|
|
|
;
|
|
; Process the DPC List.
|
|
;
|
|
|
|
|
|
rdl10: ;
|
|
|
|
ifndef NT_UP
|
|
|
|
ACQUIRE_SPINLOCK esi, rdl50, NoChecking ; acquire DPC lock
|
|
cmp ebp, [ebp].LsFlink ; check if DPC list is empty
|
|
je rdl45 ; if eq, DPC list is empty
|
|
|
|
endif
|
|
|
|
mov edx, [ebp].LsFlink ; get address of next entry
|
|
mov ecx, [edx].LsFlink ; get address of next entry
|
|
mov [ebp].LsFlink, ecx ; set address of next in header
|
|
mov [ecx].LsBlink, ebp ; set address of previous in next
|
|
sub edx, DpDpcListEntry ; compute address of DPC object
|
|
mov ecx, [edx].DpDeferredRoutine ; get DPC routine address
|
|
if DBG
|
|
|
|
push edi ; save register
|
|
mov edi, esp ; save current stack pointer
|
|
|
|
endif
|
|
|
|
|
|
FPOFRAME ?FpoValue,0
|
|
|
|
push [edx].DpSystemArgument2 ; second system argument
|
|
push [edx].DpSystemArgument1 ; first system argument
|
|
push [edx].DpDeferredContext ; get deferred context argument
|
|
push edx ; address of DPC object
|
|
mov dword ptr [edx]+DpLock, 0 ; clear DPC inserted state
|
|
dec dword ptr [ebx].PcPrcbData.PbDpcQueueDepth ; decrement depth
|
|
if DBG
|
|
mov PCR[PcPrcbData.PbDebugDpcTime], 0 ; Reset the time in DPC
|
|
endif
|
|
|
|
ifndef NT_UP
|
|
|
|
RELEASE_SPINLOCK esi, NoChecking ; release DPC lock
|
|
|
|
endif
|
|
|
|
sti ; enable interrupts
|
|
|
|
cmp [esp+LOCAL_OFFSET+28], 0 ; Are we doing event tracing?
|
|
jne rdl80
|
|
rdl20:
|
|
|
|
CAPSTART <KiRetireDpcList,ecx>
|
|
call ecx ; call DPC routine
|
|
CAPEND <KiRetireDpcList>
|
|
|
|
cmp [esp+LOCAL_OFFSET+12], 0 ; Are we doing event tracing?
|
|
jne rdl90
|
|
rdl25:
|
|
|
|
if DBG
|
|
|
|
stdCall _KeGetCurrentIrql ; get current IRQL
|
|
cmp al, DISPATCH_LEVEL ; check if still at dispatch level
|
|
jne rdl55 ; if ne, not at dispatch level
|
|
cmp esp, edi ; check if stack pointer is correct
|
|
jne rdl60 ; if ne, stack pointer is not correct
|
|
rdl30: pop edi ; restore register
|
|
|
|
endif
|
|
|
|
FPOFRAME ?FpoValue,0
|
|
|
|
rdl35: cli ; disable interrupts
|
|
cmp ebp, [ebp].LsFlink ; check if DPC list is empty
|
|
jne rdl10 ; if ne, DPC list not empty
|
|
|
|
;
|
|
; Clear DPC routine active and DPC requested flags.
|
|
;
|
|
|
|
rdl40: mov [ebx].PcPrcbData.PbDpcRoutineActive, 0
|
|
mov [ebx].PcPrcbData.PbDpcInterruptRequested, 0
|
|
|
|
;
|
|
; Check one last time that the DPC list is empty. This is required to
|
|
; close a race condition with the DPC queuing code where it appears that
|
|
; a DPC routine is active (and thus an interrupt is not requested), but
|
|
; this code has decided the DPC list is empty and is clearing the DPC
|
|
; active flag.
|
|
;
|
|
|
|
cmp ebp, [ebp].LsFlink ; check if DPC list is empty
|
|
jne rdl5 ; if ne, DPC list not empty
|
|
|
|
add esp, 16 ; pop locals
|
|
|
|
ifndef NT_UP
|
|
|
|
pop esi ; retore register
|
|
|
|
endif
|
|
ret ; return
|
|
|
|
;
|
|
; Unlock DPC list and clear DPC active.
|
|
;
|
|
|
|
rdl45: ;
|
|
|
|
ifndef NT_UP
|
|
|
|
RELEASE_SPINLOCK esi, NoChecking ; release DPC lock
|
|
jmp short rdl40 ;
|
|
|
|
endif
|
|
|
|
ifndef NT_UP
|
|
|
|
rdl50: sti ; enable interrupts
|
|
SPIN_ON_SPINLOCK esi, <short rdl35> ; spin until lock is freee
|
|
|
|
endif
|
|
|
|
|
|
if DBG
|
|
|
|
rdl55: stdCall _KeBugCheckEx, <IRQL_NOT_GREATER_OR_EQUAL, ebx, eax, 0, 0> ;
|
|
|
|
rdl60: push dword ptr [edi+12] ; push address of DPC function
|
|
push offset FLAT:_MsgDpcTrashedEsp ; push message address
|
|
call _DbgPrint ; print debug message
|
|
add esp, 8 ; remove arguments from stack
|
|
int 3 ; break into debugger
|
|
mov esp, edi ; reset stack pointer
|
|
jmp rdl30 ;
|
|
|
|
endif
|
|
|
|
;
|
|
; Check if logging is on. If so, set logical on stack.
|
|
;
|
|
rdl70:
|
|
mov eax, _PPerfGlobalGroupMask ; Load the ptr into eax
|
|
cmp eax, 0
|
|
jz rdl3
|
|
test dword ptr [eax+PERF_DPC_OFFSET], PERF_DPC_FLAG
|
|
jz rdl3 ; return if our flag is not set
|
|
|
|
mov [esp+12], 1 ; indicate DPC logging is on
|
|
jmp rdl3
|
|
|
|
;
|
|
; If logging DPC info, grab a timestamp to calculate the time in
|
|
; the service routine.
|
|
;
|
|
rdl80:
|
|
push ecx ; save DpcRoutine
|
|
PERF_GET_TIMESTAMP ; Places 64bit in edx:eax and trashes ecx
|
|
pop ecx
|
|
|
|
mov [esp+LOCAL_OFFSET+16], eax
|
|
mov [esp+LOCAL_OFFSET+20], edx
|
|
mov edx, [esp]
|
|
mov [esp+LOCAL_OFFSET+24], ecx ; Saves the service routine address
|
|
|
|
jmp rdl20
|
|
|
|
|
|
;
|
|
; Log the service routine and inital timestamp
|
|
;
|
|
rdl90:
|
|
mov eax, [esp+LOCAL_OFFSET] ; pass the initial time
|
|
mov edx, [esp+LOCAL_OFFSET+4]
|
|
push edx
|
|
push eax
|
|
mov ecx, [esp+LOCAL_OFFSET+16] ; load saved service routine address
|
|
fstCall PerfInfoLogDpc
|
|
|
|
jmp rdl25
|
|
|
|
KiRetireDpcList endp
|
|
|
|
ifdef DBGMP
|
|
cPublicProc _KiPollDebugger,0
|
|
cPublicFpo 0,3
|
|
push eax
|
|
push ecx
|
|
push edx
|
|
POLL_DEBUGGER
|
|
pop edx
|
|
pop ecx
|
|
pop eax
|
|
stdRET _KiPollDebugger
|
|
stdENDP _KiPollDebugger
|
|
|
|
endif
|
|
|
|
page , 132
|
|
subttl "Adjust TSS ESP0 value"
|
|
;++
|
|
;
|
|
; VOID
|
|
; KiAdjustEsp0 (
|
|
; IN PKTRAP_FRAME TrapFrame
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This routine puts the apropriate ESP0 value in the esp0 field of the
|
|
; TSS. This allows protect mode and V86 mode to use the same stack
|
|
; frame. The ESP0 value for protected mode is 16 bytes lower than
|
|
; for V86 mode to compensate for the missing segment registers.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; TrapFrame - Supplies a pointer to the TrapFrame.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
cPublicProc _Ki386AdjustEsp0 ,1
|
|
|
|
stdCall ___KeGetCurrentThread
|
|
|
|
mov edx,[esp + 4] ; edx -> trap frame
|
|
mov eax,[eax]+thInitialStack ; eax = base of stack
|
|
test dword ptr [edx]+TsEFlags,EFLAGS_V86_MASK ; is this a V86 frame?
|
|
jnz short ae10
|
|
|
|
sub eax,TsV86Gs - TsHardwareSegSS ; compensate for missing regs
|
|
ae10: sub eax,NPX_FRAME_LENGTH
|
|
pushfd ; Make sure we don't move
|
|
cli ; processors while we do this
|
|
mov edx,PCR[PcTss]
|
|
mov [edx]+TssEsp0,eax ; set Esp0 value
|
|
popfd
|
|
stdRET _Ki386AdjustEsp0
|
|
|
|
stdENDP _Ki386AdjustEsp0
|
|
|
|
|
|
_TEXT$00 ends
|
|
end
|