1393 lines
34 KiB
NASM
1393 lines
34 KiB
NASM
if NT_INST
|
|
|
|
else
|
|
TITLE "Spin Locks"
|
|
;++
|
|
;
|
|
; Copyright (c) 1989-1998 Microsoft Corporation
|
|
;
|
|
; Module Name:
|
|
;
|
|
; spinlock.asm
|
|
;
|
|
; Abstract:
|
|
;
|
|
; This module implements x86 spinlock functions for the PC+MP HAL.
|
|
;
|
|
; Author:
|
|
;
|
|
; Bryan Willman (bryanwi) 13 Dec 89
|
|
;
|
|
; Environment:
|
|
;
|
|
; Kernel mode only.
|
|
;
|
|
; Revision History:
|
|
;
|
|
; Ron Mosgrove (o-RonMo) Dec 93 - modified for PC+MP HAL.
|
|
;--
|
|
|
|
PAGE
|
|
|
|
.486p
|
|
|
|
include callconv.inc ; calling convention macros
|
|
include i386\kimacro.inc
|
|
include hal386.inc
|
|
include mac386.inc
|
|
include apic.inc
|
|
include ntapic.inc
|
|
|
|
EXTRNP _KeBugCheckEx,5,IMPORT
|
|
EXTRNP KfRaiseIrql, 1,,FASTCALL
|
|
EXTRNP KfLowerIrql, 1,,FASTCALL
|
|
EXTRNP _KeSetEventBoostPriority, 2, IMPORT
|
|
EXTRNP _KeWaitForSingleObject,5, IMPORT
|
|
extrn _HalpVectorToIRQL:byte
|
|
extrn _HalpIRQLtoTPR:byte
|
|
|
|
ifdef NT_UP
|
|
LOCK_ADD equ add
|
|
LOCK_DEC equ dec
|
|
LOCK_CMPXCHG equ cmpxchg
|
|
else
|
|
LOCK_ADD equ lock add
|
|
LOCK_DEC equ lock dec
|
|
LOCK_CMPXCHG equ lock cmpxchg
|
|
endif
|
|
|
|
|
|
_TEXT SEGMENT PARA PUBLIC 'CODE'
|
|
ASSUME DS:FLAT, ES:FLAT, SS:FLAT, FS:NOTHING, GS:NOTHING
|
|
|
|
PAGE
|
|
SUBTTL "Acquire Kernel Spin Lock"
|
|
;++
|
|
;
|
|
; KIRQL
|
|
; FASTCALL
|
|
; KfAcquireSpinLock (
|
|
; IN PKSPIN_LOCK SpinLock
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function raises to DISPATCH_LEVEL and then acquires a the
|
|
; kernel spin lock.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; OldIrql (TOS+8) - pointer to place old irql.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall KfAcquireSpinLock ,1
|
|
cPublicFpo 0,0
|
|
|
|
mov edx, dword ptr APIC[LU_TPR] ; (edx) = Old Priority (Vector)
|
|
mov dword ptr APIC[LU_TPR], DPC_VECTOR ; Write New Priority to the TPR
|
|
|
|
shr edx, 4
|
|
movzx eax, _HalpVectorToIRQL[edx] ; (al) = OldIrql
|
|
|
|
ifndef NT_UP
|
|
|
|
;
|
|
; Attempt to assert the lock
|
|
;
|
|
|
|
sl10: ACQUIRE_SPINLOCK ecx,<short sl20>
|
|
fstRET KfAcquireSpinLock
|
|
|
|
;
|
|
; Lock is owned, spin till it looks free, then go get it again.
|
|
;
|
|
|
|
align dword
|
|
|
|
sl20: SPIN_ON_SPINLOCK ecx,sl10
|
|
endif
|
|
|
|
|
|
fstRET KfAcquireSpinLock
|
|
fstENDP KfAcquireSpinLock
|
|
|
|
|
|
PAGE
|
|
SUBTTL "Acquire Kernel Spin Lock"
|
|
;++
|
|
;
|
|
; KIRQL
|
|
; FASTCALL
|
|
; KeAcquireSpinLockRaiseToSynch (
|
|
; IN PKSPIN_LOCK SpinLock
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function acquires the SpinLock at SYNCH_LEVEL. The function
|
|
; is optimized for hotter locks (the lock is tested before acquiring,
|
|
; and any spin occurs at OldIrql).
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; OldIrql (TOS+8) - pointer to place old irql.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall KeAcquireSpinLockRaiseToSynch,1
|
|
cPublicFpo 0,0
|
|
|
|
mov edx, dword ptr APIC[LU_TPR] ; (ecx) = Old Priority (Vector)
|
|
mov eax, edx
|
|
shr eax, 4
|
|
movzx eax, _HalpVectorToIRQL[eax] ; (al) = OldIrql
|
|
|
|
ifdef NT_UP
|
|
mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR ; Write New Priority to the TPR
|
|
fstRET KeAcquireSpinLockRaiseToSynch
|
|
else
|
|
|
|
;
|
|
; Test lock
|
|
;
|
|
|
|
TEST_SPINLOCK ecx,<short sls30>
|
|
|
|
;
|
|
; Raise irql.
|
|
;
|
|
|
|
sls10: mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR
|
|
|
|
;
|
|
; Attempt to assert the lock
|
|
;
|
|
|
|
ACQUIRE_SPINLOCK ecx,<short sls20>
|
|
fstRET KeAcquireSpinLockRaiseToSynch
|
|
|
|
;
|
|
; Lock is owned, spin till it looks free, then go get it
|
|
;
|
|
|
|
align dword
|
|
|
|
sls20: mov dword ptr APIC[LU_TPR], edx
|
|
|
|
align dword
|
|
sls30: SPIN_ON_SPINLOCK ecx,sls10
|
|
endif
|
|
|
|
fstENDP KeAcquireSpinLockRaiseToSynch
|
|
|
|
|
|
|
|
ifndef NT_UP
|
|
;++
|
|
;
|
|
; KIRQL
|
|
; FASTCALL
|
|
; KeAcquireSpinLockRaiseToSynchMCE (
|
|
; IN PKSPIN_LOCK SpinLock
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function performs the same function as KeAcquireSpinLockRaiseToSynch
|
|
; but provides a work around for an IFU errata for Pentium Pro processors
|
|
; prior to stepping 619.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; OldIrql (TOS+8) - pointer to place old irql.
|
|
;
|
|
;--
|
|
|
|
align 16
|
|
cPublicFastCall KeAcquireSpinLockRaiseToSynchMCE,1
|
|
cPublicFpo 0,0
|
|
|
|
mov edx, dword ptr APIC[LU_TPR] ; (ecx) = Old Priority (Vector)
|
|
mov eax, edx
|
|
shr eax, 4
|
|
movzx eax, _HalpVectorToIRQL[eax] ; (al) = OldIrql
|
|
|
|
;
|
|
; Test lock
|
|
;
|
|
; TEST_SPINLOCK ecx,<short slm30> ; NOTE - Macro expanded below:
|
|
|
|
test dword ptr [ecx], 1
|
|
nop ; On a P6 prior to stepping B1 (619), we
|
|
nop ; need these 5 NOPs to ensure that we
|
|
nop ; do not take a machine check exception.
|
|
nop ; The cost is just 1.5 clocks as the P6
|
|
nop ; just tosses the NOPs.
|
|
|
|
jnz short slm30
|
|
|
|
;
|
|
; Raise irql.
|
|
;
|
|
|
|
slm10: mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR
|
|
|
|
;
|
|
; Attempt to assert the lock
|
|
;
|
|
|
|
ACQUIRE_SPINLOCK ecx,<short slm20>
|
|
fstRET KeAcquireSpinLockRaiseToSynchMCE
|
|
|
|
;
|
|
; Lock is owned, spin till it looks free, then go get it
|
|
;
|
|
|
|
align dword
|
|
|
|
slm20: mov dword ptr APIC[LU_TPR], edx
|
|
|
|
align dword
|
|
slm30: SPIN_ON_SPINLOCK ecx,slm10
|
|
|
|
fstENDP KeAcquireSpinLockRaiseToSynchMCE
|
|
endif
|
|
|
|
|
|
PAGE
|
|
SUBTTL "Release Kernel Spin Lock"
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; KeReleaseSpinLock (
|
|
; IN PKSPIN_LOCK SpinLock,
|
|
; IN KIRQL NewIrql
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function releases a kernel spin lock and lowers to the new irql.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock.
|
|
; (dl) = NewIrql - New irql value to set.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
align 16
|
|
cPublicFastCall KfReleaseSpinLock ,2
|
|
cPublicFpo 0,0
|
|
xor eax, eax
|
|
mov al, dl ; (eax) = new irql value
|
|
|
|
ifndef NT_UP
|
|
RELEASE_SPINLOCK ecx ; release spinlock
|
|
endif
|
|
|
|
xor ecx, ecx
|
|
mov cl, _HalpIRQLtoTPR[eax] ; get TPR value corresponding to IRQL
|
|
mov dword ptr APIC[LU_TPR], ecx
|
|
|
|
;
|
|
; We have to ensure that the requested priority is set before
|
|
; we return. The caller is counting on it.
|
|
;
|
|
|
|
mov eax, dword ptr APIC[LU_TPR]
|
|
if DBG
|
|
cmp ecx, eax ; Verify IRQL read back is same as
|
|
je short @f ; set value
|
|
int 3
|
|
@@:
|
|
endif
|
|
fstRET KfReleaseSpinLock
|
|
|
|
fstENDP KfReleaseSpinLock
|
|
|
|
;++
|
|
;
|
|
; KIRQL
|
|
; FASTCALL
|
|
; HalpAcquireHighLevelLock (
|
|
; IN PKSPIN_LOCK SpinLock
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; Acquires a spinlock with interrupts disabled.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; OldIrql (TOS+8) - pointer to place old irql.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall HalpAcquireHighLevelLock ,1
|
|
pushfd
|
|
pop eax
|
|
|
|
ahll10: cli
|
|
ACQUIRE_SPINLOCK ecx, ahll20
|
|
fstRET HalpAcquireHighLevelLock
|
|
|
|
ahll20:
|
|
push eax
|
|
popfd
|
|
|
|
SPIN_ON_SPINLOCK ecx, <ahll10>
|
|
|
|
fstENDP HalpAcquireHighLevelLock
|
|
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; HalpReleaseHighLevelLock (
|
|
; IN PKSPIN_LOCK SpinLock,
|
|
; IN KIRQL NewIrql
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function releases a kernel spin lock and lowers to the new irql.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = SpinLock - Supplies a pointer to a kernel spin lock.
|
|
; (dl) = NewIrql - New irql value to set.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall HalpReleaseHighLevelLock ,2
|
|
|
|
RELEASE_SPINLOCK ecx
|
|
push edx
|
|
popfd
|
|
fstRET HalpReleaseHighLevelLock
|
|
|
|
fstENDP HalpReleaseHighLevelLock
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; ExAcquireFastMutex (
|
|
; IN PFAST_MUTEX FastMutex
|
|
; )
|
|
;
|
|
; Routine description:
|
|
;
|
|
; This function acquires ownership of the specified FastMutex.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = FastMutex - Supplies a pointer to the fast mutex.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall ExAcquireFastMutex,1
|
|
cPublicFpo 0,0
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector)
|
|
|
|
if DBG
|
|
;
|
|
; Caller must already be at or below APC_LEVEL.
|
|
;
|
|
|
|
cmp eax, APC_VECTOR
|
|
jg short afm11 ; irql too high ==> fatal.
|
|
endif
|
|
|
|
mov dword ptr APIC[LU_TPR], APC_VECTOR ; Write New Priority to the TPR
|
|
|
|
LOCK_DEC dword ptr [ecx].FmCount ; Get count
|
|
jz short afm_ret ; The owner? Yes, Done
|
|
|
|
inc dword ptr [ecx].FmContention
|
|
|
|
cPublicFpo 0,2
|
|
push ecx
|
|
push eax
|
|
add ecx, FmEvent ; Wait on Event
|
|
stdCall _KeWaitForSingleObject,<ecx,WrExecutive,0,0,0>
|
|
pop eax ; (al) = OldTpr
|
|
pop ecx ; (ecx) = FAST_MUTEX
|
|
|
|
cPublicFpo 0,0
|
|
afm_ret:
|
|
mov byte ptr [ecx].FmOldIrql, al
|
|
|
|
;
|
|
; Use esp to track the owning thread for debugging purposes.
|
|
; !thread from kd will find the owning thread. Note that the
|
|
; owner isn't cleared on release, check if the mutex is owned
|
|
; first.
|
|
;
|
|
|
|
mov dword ptr [ecx].FmOwner, esp
|
|
fstRet ExAcquireFastMutex
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,1
|
|
afm11: stdCall _KeBugCheckEx,<IRQL_NOT_GREATER_OR_EQUAL,ecx,eax,033h,0>
|
|
|
|
endif
|
|
|
|
fstENDP ExAcquireFastMutex
|
|
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; FASTCALL
|
|
; ExReleaseFastMutex (
|
|
; IN PFAST_MUTEX FastMutex
|
|
; )
|
|
;
|
|
; Routine description:
|
|
;
|
|
; This function releases ownership of the FastMutex.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = FastMutex - Supplies a pointer to the fast mutex.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall ExReleaseFastMutex,1
|
|
cPublicFpo 0,0
|
|
|
|
if DBG
|
|
;
|
|
; Caller must already be at APC_LEVEL or have APCs blocked.
|
|
;
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector)
|
|
cmp eax, APC_VECTOR
|
|
je short rfm04 ; irql is ok.
|
|
|
|
if 0
|
|
mov eax, PCR[PcPrcb]
|
|
mov eax, [eax].PbCurrentThread ; (eax) = Current Thread
|
|
|
|
cmp dword ptr [eax]+ThKernelApcDisable, 0
|
|
jne short rfm04 ; APCs disabled, this is ok
|
|
|
|
cmp dword ptr [eax]+ThTeb, 0
|
|
je short rfm04 ; No TEB ==> system thread, this is ok
|
|
|
|
test dword ptr [eax]+ThTeb, 080000000h
|
|
jnz short rfm04 ; TEB in system space, this is ok
|
|
endif
|
|
|
|
jmp short rfm20
|
|
rfm04:
|
|
endif
|
|
xor eax, eax
|
|
mov al, byte ptr [ecx].FmOldIrql ; (eax) = OldTpr
|
|
|
|
LOCK_ADD dword ptr [ecx].FmCount, 1 ; Remove our count
|
|
js short rfm05 ; if < 0, set event
|
|
jnz short rfm10 ; if != 0, don't set event
|
|
|
|
cPublicFpo 0,1
|
|
rfm05: add ecx, FmEvent
|
|
push eax ; save new tpr
|
|
stdCall _KeSetEventBoostPriority, <ecx, 0>
|
|
pop eax ; restore tpr
|
|
|
|
cPublicFpo 0,0
|
|
rfm10: mov dword ptr APIC[LU_TPR], eax
|
|
mov ecx, dword ptr APIC[LU_TPR]
|
|
if DBG
|
|
cmp eax, ecx ; Verify TPR is what was
|
|
je short @f ; written
|
|
int 3
|
|
@@:
|
|
endif
|
|
fstRet ExReleaseFastMutex
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,1
|
|
rfm20: stdCall _KeBugCheckEx,<IRQL_NOT_GREATER_OR_EQUAL,ecx,eax,034h,0>
|
|
|
|
endif
|
|
|
|
fstENDP ExReleaseFastMutex
|
|
|
|
|
|
;++
|
|
;
|
|
; BOOLEAN
|
|
; FASTCALL
|
|
; ExTryToAcquireFastMutex (
|
|
; IN PFAST_MUTEX FastMutex
|
|
; )
|
|
;
|
|
; Routine description:
|
|
;
|
|
; This function acquires ownership of the FastMutex.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; (ecx) = FastMutex - Supplies a pointer to the fast mutex.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; Returns TRUE if the FAST_MUTEX was acquired; otherwise FALSE.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall ExTryToAcquireFastMutex,1
|
|
cPublicFpo 0,0
|
|
|
|
if DBG
|
|
;
|
|
; Caller must already be at or below APC_LEVEL.
|
|
;
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector)
|
|
cmp eax, APC_VECTOR
|
|
jg short tam11 ; irql too high ==> fatal.
|
|
endif
|
|
|
|
;
|
|
; Try to acquire.
|
|
;
|
|
|
|
cmp dword ptr [ecx].FmCount, 1 ; Busy?
|
|
jne short tam25 ; Yes, abort
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority (Vector)
|
|
mov dword ptr APIC[LU_TPR], APC_VECTOR ; Write New Priority to the TPR
|
|
|
|
cPublicFpo 0,1
|
|
push eax ; Save Old TPR
|
|
|
|
mov edx, 0 ; Value to set
|
|
mov eax, 1 ; Value to compare against
|
|
LOCK_CMPXCHG dword ptr [ecx].FmCount, edx ; Attempt to acquire
|
|
jnz short tam20 ; got it?
|
|
|
|
cPublicFpo 0,0
|
|
pop edx ; (edx) = Old TPR
|
|
mov eax, 1 ; return TRUE
|
|
mov byte ptr [ecx].FmOldIrql, dl ; Store Old TPR
|
|
fstRet ExTryToAcquireFastMutex
|
|
|
|
tam20: pop ecx ; (ecx) = Old TPR
|
|
mov dword ptr APIC[LU_TPR], ecx
|
|
mov eax, dword ptr APIC[LU_TPR]
|
|
|
|
if DBG
|
|
cmp ecx, eax ; Verify TPR is what was
|
|
je short @f ; written
|
|
int 3
|
|
@@:
|
|
endif
|
|
|
|
tam25: xor eax, eax ; return FALSE
|
|
YIELD
|
|
fstRet ExTryToAcquireFastMutex ; all done
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,1
|
|
tam11: stdCall _KeBugCheckEx,<IRQL_NOT_GREATER_OR_EQUAL,ecx,eax,033h,0>
|
|
|
|
endif
|
|
|
|
fstENDP ExTryToAcquireFastMutex
|
|
|
|
|
|
page ,132
|
|
subttl "Acquire Queued SpinLock Raise to Synch"
|
|
|
|
; compile time assert sizeof(KSPIN_LOCK_QUEUE) == 8
|
|
|
|
.errnz (LOCK_QUEUE_HEADER_SIZE - 8)
|
|
|
|
align 16
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; KeAcquireInStackQueuedSpinLock (
|
|
; IN PKSPIN_LOCK SpinLock,
|
|
; IN PKLOCK_QUEUE_HANDLE LockHandle
|
|
; )
|
|
;
|
|
; VOID
|
|
; KeAcquireInStackQueuedSpinLockRaiseToSynch (
|
|
; IN PKSPIN_LOCK SpinLock,
|
|
; IN PKLOCK_QUEUE_HANDLE LockHandle
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; KeAcquireInStackQueuedSpinLock...
|
|
;
|
|
; The Kx versions use a LOCK_QUEUE_HANDLE structure rather than
|
|
; LOCK_QUEUE structures in the PRCB. Old IRQL is stored in the
|
|
; LOCK_QUEUE_HANDLE.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; SpinLock (ecx) Address of Actual Lock.
|
|
; LockHandle (edx) Address of lock context.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None. Actually returns OldIrql because common code is used
|
|
; for all implementations.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall KeAcquireInStackQueuedSpinLockRaiseToSynch,2
|
|
cPublicFpo 0,0
|
|
|
|
|
|
ifdef NT_UP
|
|
|
|
; In the Uniprocessor case, just raise IRQL to SYNCH
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority
|
|
shr eax, 4
|
|
mov al, byte ptr _HalpVectorToIRQL[eax] ; (al) = OldIrql
|
|
mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR
|
|
mov [edx].LqhOldIrql, al ; save old IRQL in lock handle
|
|
|
|
fstRET KeAcquireInStackQueuedSpinLockRaiseToSynch
|
|
|
|
else
|
|
|
|
; MP case, use KeAcquireInStackQueuedSpinLock to get the lock and raise
|
|
; to SYNCH asap afterwards.
|
|
|
|
call @KeAcquireInStackQueuedSpinLock@8
|
|
mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR
|
|
|
|
fstRET KeAcquireInStackQueuedSpinLockRaiseToSynch
|
|
|
|
endif
|
|
|
|
fstENDP KeAcquireInStackQueuedSpinLockRaiseToSynch
|
|
|
|
|
|
|
|
|
|
|
|
cPublicFastCall KeAcquireInStackQueuedSpinLock,2
|
|
cPublicFpo 0,0
|
|
|
|
|
|
; Raise IRQL to DISPATCH_LEVEL
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority
|
|
shr eax, 4
|
|
mov al, byte ptr _HalpVectorToIRQL[eax] ; (al) = OldIrql
|
|
mov dword ptr APIC[LU_TPR], DPC_VECTOR ; raise to DISPATCH level
|
|
mov [edx].LqhOldIrql, al ; save old IRQL in lock handle
|
|
|
|
ifdef NT_UP
|
|
|
|
fstRET KeAcquireInStackQueuedSpinLock
|
|
|
|
else
|
|
|
|
; Save actual lock address in lock handle.
|
|
|
|
mov [edx].LqhLock, ecx
|
|
mov dword ptr [edx].LqhNext, 0
|
|
|
|
|
|
; ecx contains the address of the actual lock
|
|
; and edx the address of a queued lock structure.
|
|
|
|
cPublicFpo 0,1
|
|
mov eax, edx ; save Lock Queue entry address
|
|
|
|
if DBG
|
|
|
|
test ecx, LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT
|
|
jnz short iqsl98 ; jiff lock already held (or
|
|
; this proc already waiting).
|
|
endif
|
|
|
|
; Exchange the value of the lock with the address of this
|
|
; Lock Queue entry.
|
|
|
|
xchg [ecx], edx
|
|
|
|
cmp edx, 0 ; check if lock is held
|
|
jnz short iqsl40 ; jiff held
|
|
|
|
; lock has been acquired.
|
|
|
|
cPublicFpo 0,0
|
|
|
|
; note: the actual lock address is word aligned, we use
|
|
; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT,
|
|
; bit 1 is LOCK_QUEUE_OWNER.
|
|
|
|
or ecx, LOCK_QUEUE_OWNER ; mark self as lock owner
|
|
mov [eax].LqLock, ecx
|
|
|
|
iqsl20:
|
|
fstRET KeAcquireInStackQueuedSpinLock
|
|
|
|
cPublicFpo 0,1
|
|
|
|
iqsl40:
|
|
; The lock is already held by another processor. Set the wait
|
|
; bit in this processor's Lock Queue entry, then set the next
|
|
; field in the Lock Queue entry of the last processor to attempt
|
|
; to acquire the lock (this is the address returned by the xchg
|
|
; above) to point to THIS processor's lock queue entry.
|
|
|
|
or ecx, LOCK_QUEUE_WAIT ; set waiting bit
|
|
mov [eax].LqLock, ecx
|
|
|
|
mov [edx].LqNext, eax ; set previous acquirer's
|
|
; next field.
|
|
; Wait.
|
|
@@:
|
|
test [eax].LqLock, LOCK_QUEUE_WAIT ; check if still waiting
|
|
jz short iqsl20 ; jif lock acquired
|
|
YIELD ; fire avoidance.
|
|
jmp short @b ; else, continue waiting
|
|
; Wait.
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,1
|
|
iqsl98: stdCall _KeBugCheckEx,<SPIN_LOCK_ALREADY_OWNED,ecx,edx,0,0>
|
|
int 3 ; so stacktrace works
|
|
|
|
endif
|
|
|
|
endif
|
|
|
|
fstENDP KeAcquireInStackQueuedSpinLock
|
|
|
|
|
|
;++
|
|
;
|
|
; KIRQL
|
|
; KeAcquireQueuedSpinLockRaiseToSynch (
|
|
; IN KSPIN_LOCK_QUEUE_NUMBER Number
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function raises the current IRQL to DISPATCH/SYNCH level
|
|
; and acquires the specified queued spinlock.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number (ecx) - Supplies the queued spinlock number.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; The previous IRQL is returned as the function value.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall KeAcquireQueuedSpinLockRaiseToSynch,1
|
|
cPublicFpo 0,0
|
|
|
|
|
|
ifdef NT_UP
|
|
|
|
; In the Uniprocessor case, just raise IRQL to SYNCH
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority
|
|
shr eax, 4
|
|
movzx eax, byte ptr _HalpVectorToIRQL[eax] ; (al) = OldIrql
|
|
mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR
|
|
|
|
fstRET KeAcquireQueuedSpinLockRaiseToSynch
|
|
|
|
else
|
|
|
|
; MP case, use KeAcquireQueuedSpinLock to get the lock and raise
|
|
; to SYNCH asap afterwards.
|
|
|
|
call @KeAcquireQueuedSpinLock@4
|
|
mov dword ptr APIC[LU_TPR], APIC_SYNCH_VECTOR
|
|
|
|
fstRET KeAcquireQueuedSpinLockRaiseToSynch
|
|
|
|
endif
|
|
|
|
fstENDP KeAcquireQueuedSpinLockRaiseToSynch
|
|
|
|
|
|
page ,132
|
|
subttl "Acquire Queued SpinLock"
|
|
|
|
;++
|
|
;
|
|
; KIRQL
|
|
; KeAcquireQueuedSpinLock (
|
|
; IN KSPIN_LOCK_QUEUE_NUMBER Number
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function raises the current IRQL to DISPATCH level
|
|
; and acquires the specified queued spinlock.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number (ecx) - Supplies the queued spinlock number.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; The previous IRQL is returned as the function value.
|
|
;
|
|
;--
|
|
|
|
|
|
cPublicFastCall KeAcquireQueuedSpinLock,1
|
|
cPublicFpo 0,0
|
|
|
|
; Get old priority (vector) from Local APIC's Task Priority
|
|
; Register and set the new priority.
|
|
|
|
mov eax, dword ptr APIC[LU_TPR] ; (eax) = Old Priority
|
|
shr eax, 4
|
|
movzx eax, byte ptr _HalpVectorToIRQL[eax] ; (al) = OldIrql
|
|
mov dword ptr APIC[LU_TPR], DPC_VECTOR ; raise to DISPATCH level
|
|
|
|
ifdef NT_UP
|
|
|
|
; in the Uniprocessor version all we do is raise IRQL.
|
|
|
|
|
|
fstRET KeAcquireQueuedSpinLock
|
|
|
|
else
|
|
|
|
; Get address of Lock Queue entry
|
|
|
|
mov edx, PCR[PcPrcb] ; get address of PRCB
|
|
lea edx, [edx+ecx*8].PbLockQueue ; get &PRCB->LockQueue[Number]
|
|
|
|
; Get address of the actual lock.
|
|
|
|
mov ecx, [edx].LqLock
|
|
aqsl10:
|
|
push eax ; save old IRQL
|
|
cPublicFpo 0,1
|
|
mov eax, edx ; save Lock Queue entry address
|
|
|
|
if DBG
|
|
|
|
test ecx, LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT
|
|
jnz short aqsl98 ; jiff lock already held (or
|
|
; this proc already waiting).
|
|
endif
|
|
|
|
; Exchange the value of the lock with the address of this
|
|
; Lock Queue entry.
|
|
|
|
xchg [ecx], edx
|
|
|
|
cmp edx, 0 ; check if lock is held
|
|
jnz short aqsl40 ; jiff held
|
|
|
|
; lock has been acquired.
|
|
|
|
cPublicFpo 0,0
|
|
|
|
; note: the actual lock address is word aligned, we use
|
|
; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT,
|
|
; bit 1 is LOCK_QUEUE_OWNER.
|
|
|
|
or ecx, LOCK_QUEUE_OWNER ; mark self as lock owner
|
|
mov [eax].LqLock, ecx
|
|
|
|
aqsl20:
|
|
pop eax ; return old IRQL
|
|
|
|
fstRET KeAcquireQueuedSpinLock
|
|
|
|
cPublicFpo 0,1
|
|
|
|
aqsl40:
|
|
; The lock is already held by another processor. Set the wait
|
|
; bit in this processor's Lock Queue entry, then set the next
|
|
; field in the Lock Queue entry of the last processor to attempt
|
|
; to acquire the lock (this is the address returned by the xchg
|
|
; above) to point to THIS processor's lock queue entry.
|
|
|
|
or ecx, LOCK_QUEUE_WAIT ; set waiting bit
|
|
mov [eax].LqLock, ecx
|
|
|
|
mov [edx].LqNext, eax ; set previous acquirer's
|
|
; next field.
|
|
cPublicFpo 0,0
|
|
|
|
; Wait.
|
|
@@:
|
|
test [eax].LqLock, LOCK_QUEUE_WAIT ; check if still waiting
|
|
jz short aqsl20 ; jif lock acquired
|
|
YIELD ; fire avoidance.
|
|
jmp short @b ; else, continue waiting
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,1
|
|
aqsl98: stdCall _KeBugCheckEx,<SPIN_LOCK_ALREADY_OWNED,ecx,edx,0,0>
|
|
int 3 ; so stacktrace works
|
|
|
|
endif
|
|
|
|
endif
|
|
|
|
fstENDP KeAcquireQueuedSpinLock
|
|
|
|
|
|
page ,132
|
|
subttl "Release Queued SpinLock"
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; KeReleaseInStackQueuedSpinLock (
|
|
; IN PKLOCK_QUEUE_HANDLE LockHandle
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function releases a queued spinlock and lowers the IRQL to
|
|
; its previous value.
|
|
;
|
|
; This differs from KeReleaseQueuedSpinLock in that this version
|
|
; uses a caller supplied lock context where that one uses a
|
|
; predefined lock context in the processor's PRCB.
|
|
;
|
|
; This version sets up a compatible register context and uses
|
|
; KeReleaseQueuedSpinLock to do the actual work.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; LockHandle (ecx) - Address of Lock Queue Handle structure.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall KeReleaseInStackQueuedSpinLock,1
|
|
cPublicFpo 0,0
|
|
|
|
movzx edx, byte ptr [ecx].LqhOldIrql ; get old irql
|
|
|
|
ifndef NT_UP
|
|
|
|
lea eax, [ecx].LqhNext ; get address of lock struct
|
|
jmp short rqsl10 ; continue in common code
|
|
|
|
else
|
|
|
|
; Set the IO APIC's Task Priority Register to the value
|
|
; corresponding to the new IRQL.
|
|
|
|
movzx ecx, byte ptr _HalpIRQLtoTPR[edx]
|
|
|
|
mov dword ptr APIC[LU_TPR], ecx
|
|
|
|
; Ensure that the requested priority is set before returning,
|
|
; the caller is counting on it.
|
|
|
|
mov eax, dword ptr APIC[LU_TPR]
|
|
if DBG
|
|
cmp ecx, eax ; Verify IRQL read back is same as
|
|
je short @f ; set value
|
|
int 3
|
|
@@:
|
|
endif
|
|
fstRET KeReleaseInStackQueuedSpinLock
|
|
endif
|
|
|
|
|
|
fstENDP KeReleaseInStackQueuedSpinLock
|
|
|
|
|
|
;++
|
|
;
|
|
; VOID
|
|
; KeReleaseQueuedSpinLock (
|
|
; IN KSPIN_LOCK_QUEUE_NUMBER Number,
|
|
; IN KIRQL OldIrql
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function releases a queued spinlock and lowers the IRQL to
|
|
; its previous value.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number (ecx) - Supplies the queued spinlock number.
|
|
; OldIrql (dl) - Supplies the IRQL value to lower to.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; None.
|
|
;
|
|
;--
|
|
|
|
cPublicFastCall KeReleaseQueuedSpinLock,2
|
|
cPublicFpo 0,0
|
|
|
|
.errnz (LOCK_QUEUE_OWNER - 2) ; error if not bit 1 for btr
|
|
|
|
ifndef NT_UP
|
|
|
|
; Get address of Lock Queue entry
|
|
|
|
mov eax, PCR[PcPrcb] ; get address of PRCB
|
|
|
|
endif
|
|
|
|
movzx edx, dl ; Irql = 8 bits from edx
|
|
|
|
ifndef NT_UP
|
|
|
|
lea eax, [eax+ecx*8].PbLockQueue ; get &PRCB->LockQueue[Number]
|
|
|
|
rqsl10:
|
|
push ebx ; need another register
|
|
cPublicFpo 0,1
|
|
|
|
; Clear the lock field in the Lock Queue entry.
|
|
|
|
mov ebx, [eax].LqNext
|
|
mov ecx, [eax].LqLock
|
|
|
|
; Quick check: If Lock Queue entry's Next field is not NULL,
|
|
; there is another waiter. Don't bother with ANY atomic ops
|
|
; in this case.
|
|
;
|
|
; Note: test clears CF and sets ZF appropriately, the following
|
|
; btr sets CF appropriately for the owner check.
|
|
|
|
test ebx, ebx
|
|
|
|
; clear the "I am owner" bit in the Lock entry.
|
|
|
|
btr ecx, 1 ; clear owner bit.
|
|
|
|
if DBG
|
|
|
|
jnc short rqsl98 ; bugcheck if was not set
|
|
; tests CF
|
|
endif
|
|
|
|
mov [eax].LqLock, ecx ; clear lock bit in queue entry
|
|
jnz short rqsl40 ; jif another processor waits
|
|
; tests ZF
|
|
|
|
; ebx contains zero here which will be used to set the new owner NULL
|
|
|
|
push eax ; save &PRCB->LockQueue[Number]
|
|
cPublicFpo 0,2
|
|
|
|
; Use compare exchange to attempt to clear the actual lock.
|
|
; If there are still no processors waiting for the lock when
|
|
; the compare exchange happens, the old contents of the lock
|
|
; should be the address of this lock entry (eax).
|
|
|
|
lock cmpxchg [ecx], ebx ; store 0 if no waiters
|
|
pop eax ; restore lock queue address
|
|
cPublicFpo 0,1
|
|
jnz short rqsl60 ; jif store failed
|
|
|
|
; The lock has been released. Lower IRQL and return to caller.
|
|
|
|
endif
|
|
|
|
rqsl20:
|
|
|
|
; Set the IO APIC's Task Priority Register to the value
|
|
; corresponding to the new IRQL.
|
|
|
|
movzx ecx, byte ptr _HalpIRQLtoTPR[edx]
|
|
|
|
ifndef NT_UP
|
|
|
|
pop ebx ; restore ebx
|
|
cPublicFpo 0,0
|
|
|
|
endif
|
|
|
|
mov dword ptr APIC[LU_TPR], ecx
|
|
|
|
; Ensure that the requested priority is set before returning,
|
|
; the caller is counting on it.
|
|
|
|
mov eax, dword ptr APIC[LU_TPR]
|
|
if DBG
|
|
cmp ecx, eax ; Verify IRQL read back is same as
|
|
je short @f ; set value
|
|
int 3
|
|
@@:
|
|
endif
|
|
fstRET KeReleaseQueuedSpinLock
|
|
|
|
ifndef NT_UP
|
|
|
|
cPublicFpo 0,1
|
|
|
|
; Another processor is waiting on this lock. Hand the lock
|
|
; to that processor by getting the address of its LockQueue
|
|
; entry, turning ON its owner bit and OFF its wait bit.
|
|
|
|
rqsl40: xor [ebx].LqLock, (LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT)
|
|
|
|
; Done, the other processor now owns the lock, clear the next
|
|
; field in my LockQueue entry (to preserve the order for entering
|
|
; the queue again) and proceed to lower IRQL and return.
|
|
|
|
mov [eax].LqNext, 0
|
|
jmp short rqsl20
|
|
|
|
|
|
; We get here if another processor is attempting to acquire
|
|
; the lock but had not yet updated the next field in this
|
|
; processor's Queued Lock Next field. Wait for the next
|
|
; field to be updated.
|
|
|
|
|
|
rqsl60: mov ebx, [eax].LqNext
|
|
test ebx, ebx ; check if still 0
|
|
jnz short rqsl40 ; jif Next field now set.
|
|
YIELD ; wait a bit
|
|
jmp short rqsl60 ; continue waiting
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,1
|
|
|
|
rqsl98: stdCall _KeBugCheckEx,<SPIN_LOCK_NOT_OWNED,ecx,eax,0,1>
|
|
int 3 ; so stacktrace works
|
|
|
|
endif
|
|
|
|
endif
|
|
|
|
fstENDP KeReleaseQueuedSpinLock
|
|
|
|
page ,132
|
|
subttl "Try to Acquire Queued SpinLock"
|
|
|
|
;++
|
|
;
|
|
; LOGICAL
|
|
; KeTryToAcquireQueuedSpinLock (
|
|
; IN KSPIN_LOCK_QUEUE_NUMBER Number,
|
|
; OUT PKIRQL OldIrql
|
|
; )
|
|
;
|
|
; LOGICAL
|
|
; KeTryToAcquireQueuedSpinLockRaiseToSynch (
|
|
; IN KSPIN_LOCK_QUEUE_NUMBER Number,
|
|
; OUT PKIRQL OldIrql
|
|
; )
|
|
;
|
|
; Routine Description:
|
|
;
|
|
; This function raises the current IRQL to DISPATCH/SYNCH level
|
|
; and attempts to acquire the specified queued spinlock. If the
|
|
; spinlock is already owned by another thread, IRQL is restored
|
|
; to its previous value and FALSE is returned.
|
|
;
|
|
; Arguments:
|
|
;
|
|
; Number (ecx) - Supplies the queued spinlock number.
|
|
; OldIrql (edx) - A pointer to the variable to receive the old
|
|
; IRQL.
|
|
;
|
|
; Return Value:
|
|
;
|
|
; TRUE if the lock was acquired, FALSE otherwise.
|
|
; N.B. ZF is set if FALSE returned, clear otherwise.
|
|
;
|
|
;--
|
|
|
|
|
|
align 16
|
|
cPublicFastCall KeTryToAcquireQueuedSpinLockRaiseToSynch,2
|
|
cPublicFpo 0,0
|
|
|
|
push APIC_SYNCH_VECTOR ; raise to SYNCH
|
|
jmp short taqsl10 ; continue in common code
|
|
|
|
fstENDP KeTryToAcquireQueuedSpinLockRaiseToSynch
|
|
|
|
|
|
cPublicFastCall KeTryToAcquireQueuedSpinLock,2
|
|
cPublicFpo 0,0
|
|
|
|
push DPC_VECTOR ; raise to DPC level
|
|
|
|
; Attempt to get the lock with interrupts disabled, raising
|
|
; the priority in the interrupt controller only if acquisition
|
|
; is successful.
|
|
taqsl10:
|
|
|
|
ifndef NT_UP
|
|
|
|
push edx ; save address of OldIrql
|
|
pushfd ; save interrupt state
|
|
cPublicFpo 0,3
|
|
|
|
; Get address of Lock Queue entry
|
|
|
|
cli
|
|
mov edx, PCR[PcPrcb] ; get address of PRCB
|
|
lea edx, [edx+ecx*8].PbLockQueue ; get &PRCB->LockQueue[Number]
|
|
|
|
; Get address of the actual lock.
|
|
|
|
mov ecx, [edx].LqLock
|
|
|
|
if DBG
|
|
|
|
test ecx, LOCK_QUEUE_OWNER+LOCK_QUEUE_WAIT
|
|
jnz short taqsl98 ; jiff lock already held (or
|
|
; this proc already waiting).
|
|
endif
|
|
|
|
; quick test, get out if already taken
|
|
|
|
cmp dword ptr [ecx], 0 ; check if already taken
|
|
jnz short taqsl60 ; jif already taken
|
|
xor eax, eax ; comparison value (not locked)
|
|
|
|
; Store the Lock Queue entry address in the lock ONLY if the
|
|
; current lock value is 0.
|
|
|
|
lock cmpxchg [ecx], edx
|
|
jnz short taqsl60
|
|
|
|
; Lock has been acquired.
|
|
|
|
; note: the actual lock address will be word aligned, we use
|
|
; the bottom two bits as indicators, bit 0 is LOCK_QUEUE_WAIT,
|
|
; bit 1 is LOCK_QUEUE_OWNER.
|
|
|
|
or ecx, LOCK_QUEUE_OWNER ; mark self as lock owner
|
|
mov [edx].LqLock, ecx
|
|
|
|
mov eax, [esp+8] ; get new IRQL
|
|
mov edx, [esp+4] ; get addr to save OldIrql
|
|
|
|
else
|
|
|
|
mov eax, [esp] ; get new IRQL
|
|
|
|
endif
|
|
|
|
; Raise IRQL and return success.
|
|
|
|
; Get old priority (vector) from Local APIC's Task Priority
|
|
; Register and set the new priority.
|
|
|
|
mov ecx, dword ptr APIC[LU_TPR] ; (ecx) = Old Priority
|
|
mov dword ptr APIC[LU_TPR], eax ; Set New Priority
|
|
|
|
ifndef NT_UP
|
|
|
|
popfd ; restore interrupt state
|
|
add esp, 8 ; free locals
|
|
|
|
else
|
|
|
|
add esp, 4 ; free local
|
|
endif
|
|
|
|
cPublicFpo 0,0
|
|
|
|
shr ecx, 4
|
|
movzx eax, _HalpVectorToIRQL[ecx] ; (al) = OldIrql
|
|
mov [edx], al ; save OldIrql
|
|
xor eax, eax ; return TRUE
|
|
or eax, 1
|
|
|
|
fstRET KeTryToAcquireQueuedSpinLock
|
|
|
|
ifndef NT_UP
|
|
|
|
taqsl60:
|
|
; The lock is already held by another processor. Indicate
|
|
; failure to the caller.
|
|
|
|
popfd ; restore interrupt state
|
|
add esp, 8 ; free locals
|
|
xor eax, eax ; return FALSE
|
|
fstRET KeTryToAcquireQueuedSpinLock
|
|
|
|
if DBG
|
|
|
|
cPublicFpo 0,2
|
|
|
|
taqsl98: stdCall _KeBugCheckEx,<SPIN_LOCK_ALREADY_OWNED,ecx,edx,0,0>
|
|
int 3 ; so stacktrace works
|
|
|
|
endif
|
|
|
|
endif
|
|
|
|
fstENDP KeTryToAcquireQueuedSpinLock
|
|
_TEXT ends
|
|
|
|
ENDIF ; NT_INST
|
|
|
|
end
|