able/windows-nt
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
windows-nt/Source/XPSP1/NT/base/ntos/ke/alpha/spinlock.s

1231 lines
33 KiB
ArmAsm
Raw Normal View History

Add source files
2020-09-26 03:20:57 -05:00
// TITLE("Spin Locks")
//++
//
// Copyright (c) 1990 Microsoft Corporation
// Copyright (c) 1992 Digital Equipment Corporation
//
// Module Name:
//
// spinlock.s
//
// Abstract:
//
// This module implements the routines for acquiring and releasing
// spin locks.
//
// Author:
//
// David N. Cutler (davec) 23-Mar-1990
// Joe Notarangelo 06-Apr-1992
//
// Environment:
//
// Kernel mode only.
//
// Revision History:
//
//--
#include "ksalpha.h"
//++
//
// VOID
// KeInitializeSpinLock (
// IN PKSPIN_LOCK SpinLock
// )
//
// Routine Description:
//
// This function initializes an executive spin lock.
//
// Argument:
//
// SpinLock (a0) - Supplies a pointer to the executive spin lock.
//
// Return Value:
//
// None.
//
//--
LEAF_ENTRY( KeInitializeSpinLock )
STP zero, 0(a0) // set spin lock not owned
ret zero, (ra) // return
.end KeInitializeSpinLock
//++
//
// VOID
// KeAcquireSpinLock (
// IN PKSPIN_LOCK SpinLock
// OUT PKIRQL OldIrql
// )
//
// Routine Description:
//
// This function raises the current IRQL to DISPATCH_LEVEL and acquires
// the specified executive spinlock.
//
// Arguments:
//
// SpinLock (a0) - Supplies a pointer to a executive spinlock.
//
// OldIrql (a1) - Supplies a pointer to a variable that receives the
// the previous IRQL value.
//
// Return Value:
//
// None.
//
//--
LEAF_ENTRY(KeAcquireSpinLock)
//
// Raise IRQL to DISPATCH_LEVEL and acquire the specified spinlock.
//
// N.B. The raise IRQL code is duplicated here is avoid any extra overhead
// since this is such a common operation.
//
// N.B. The previous IRQL must not be stored until the lock is owned.
//
// N.B. The longword surrounding the previous IRQL must not be read
// until the lock is owned.
//
bis a0, zero, t5 // t5 = address of spin lock
ldil a0, DISPATCH_LEVEL // set new IRQL
bis a1, zero, t0 // t0 = a1, a1 may be destroyed
SWAP_IRQL // swap irql, on return v0 = old irql
//
// Acquire the specified spinlock.
//
// N.B. code below branches forward if spinlock fails intentionally
// because branch forwards are predicted to miss
//
#if !defined(NT_UP)
10: LDP_L t3, 0(t5) // get current lock value
bis t5, zero, t4 // set ownership value
bne t3, 15f // if ne => lock owned
STP_C t4, 0(t5) // set lock owned
beq t4, 15f // if eq => stx_c failed
mb // synchronize memory access
#endif
//
// Save the old Irql at the address saved by the caller.
// Insure that the old Irql is updated with longword granularity.
//
ldq_u t1, 0(t0) // read quadword surrounding KIRQL
bic t0, 3, t2 // get address of containing longword
mskbl t1, t0, t1 // clear KIRQL byte in quadword
insbl v0, t0, v0 // get new KIRQL to correct byte
bis t1, v0, t1 // merge KIRQL into quadword
extll t1, t2, t1 // get longword containg KIRQL
stl t1, 0(t2) // store containing longword
ret zero, (ra) // return
//
// Attempt to acquire spinlock failed.
//
#if !defined(NT_UP)
15: LDP t3, 0(t5) // get current lock value
beq t3, 10b // retry acquire lock if unowned
br zero, 15b // loop in cache until lock free
#endif
.end KeAcquireSpinLock
SBTTL("Acquire SpinLock and Raise to Synch")
//++
//
// KIRQL
// KeAcquireSpinLockRaiseToSynch (
// IN PKSPIN_LOCK SpinLock
// )
//
// Routine Description:
//
// This function raises the current IRQL to synchronization level and
// acquires the specified spinlock.
//
// Arguments:
//
// SpinLock (a0) - Supplies a pointer to the spinlock that is to be
// acquired.
//
// Return Value:
//
// The previous IRQL is returned as the function value.
//
//--
LEAF_ENTRY(KeAcquireSpinLockRaiseToSynch)
#if !defined(NT_UP)
bis a0, zero, t5 // save spinlock address
ldl a0, KiSynchIrql // get synch level IRQL
//
// Raise IRQL and attempt to acquire the specified spinlock.
//
10: SWAP_IRQL // raise IRQL to synch level
LDP_L t3, 0(t5) // get current lock value
bis t5, zero, t4 // set ownership value
bne t3, 25f // if ne, lock owned
STP_C t4, 0(t5) // set lock owned
beq t4, 25f // if eq, conditional store failed
mb // synchronize subsequent reads
ret zero, (ra)
//
// Spinlock is owned, lower IRQL and spin in cache until it looks free.
//
25: bis v0, zero, a0 // get previous IRQL value
SWAP_IRQL // lower IRQL
bis v0, zero, a0 // save synch level IRQL
26: LDP t3, 0(t5) // get current lock value
beq t3, 10b // retry acquire if unowned
br zero, 26b // loop in cache until free
#else
ldl a0, KiSynchIrql // get synch level IRQL
SWAP_IRQL // rasie IRQL to synch levcel
ret zero, (ra) // return
.end KeAcquireSpinLockRaiseToSynch
#endif
//++
//
// KIRQL
// KeAcquireSpinLockRaiseToDpc (
// IN PKSPIN_LOCK SpinLock
// )
//
// Routine Description:
//
// This function raises the current IRQL to dispatcher level and acquires
// the specified spinlock.
//
// Arguments:
//
// SpinLock (a0) - Supplies a pointer to the spinlock that is to be
// acquired.
//
// Return Value:
//
// The previous IRQL is returned as the function value.
//
//--
#if !defined(NT_UP)
ALTERNATE_ENTRY(KeAcquireSpinLockRaiseToDpc)
bis a0, zero, t5 // save spinlock address
ldil a0, DISPATCH_LEVEL // set IRQL level
br 10b // finish in common code
.end KeAcquireSpinLockRaiseToSynch
#else
LEAF_ENTRY(KeAcquireSpinLockRaiseToDpc)
ldil a0, DISPATCH_LEVEL // set new IRQL
SWAP_IRQL // old irql in v0
ret zero, (ra)
.end KeAcquireSpinLockRaiseToDpc
#endif
//++
//
// VOID
// KeReleaseSpinLock (
// IN PKSPIN_LOCK SpinLock
// IN KIRQL OldIrql
// )
//
// Routine Description:
//
// This function releases an executive spin lock and lowers the IRQL
// to its previous value.
//
// Arguments:
//
// SpinLock (a0) - Supplies a pointer to an executive spin lock.
//
// OldIrql (a1) - Supplies the previous IRQL value.
//
// Return Value:
//
// None.
//
//--
LEAF_ENTRY(KeReleaseSpinLock)
//
// Release the specified spinlock.
//
#if !defined(NT_UP)
mb // synchronize memory access
STP zero, 0(a0) // set spin lock not owned
#endif
//
// Lower the IRQL to the specified level.
//
// N.B. The lower IRQL code is duplicated here is avoid any extra overhead
// since this is such a common operation.
//
10: bis a1, zero, a0 // a0 = new irql
SWAP_IRQL // change to new irql
ret zero, (ra) // return
.end KeReleaseSpinLock
//++
//
// BOOLEAN
// KeTryToAcquireSpinLock (
// IN PKSPIN_LOCK SpinLock
// OUT PKIRQL OldIrql
// )
//
// Routine Description:
//
// This function raises the current IRQL to DISPATCH_LEVEL and attempts
// to acquires the specified executive spinlock. If the spinlock can be
// acquired, then TRUE is returned. Otherwise, the IRQL is restored to
// its previous value and FALSE is returned.
//
// Arguments:
//
// SpinLock (a0) - Supplies a pointer to a executive spinlock.
//
// OldIrql (a1) - Supplies a pointer to a variable that receives the
// the previous IRQL value.
//
// Return Value:
//
// If the spin lock is acquired, then a value of TRUE is returned.
// Otherwise, a value of FALSE is returned.
//
//--
LEAF_ENTRY(KeTryToAcquireSpinLock)
//
// Raise IRQL to DISPATCH_LEVEL and try to acquire the specified spinlock.
//
// N.B. The raise IRQL code is duplicated here is avoid any extra overhead
// since this is such a common operation.
//
bis a0, zero, t5 // t5 = address of spin lock
ldil a0, DISPATCH_LEVEL // new irql
bis a1, zero, t11 // t11 = a1, a1 may be clobbered
SWAP_IRQL // a0 = new, on return v0 = old irql
//
// Try to acquire the specified spinlock.
//
// N.B. A noninterlocked test is done before the interlocked attempt. This
// allows spinning without interlocked cycles.
//
#if !defined(NT_UP)
LDP t0, 0(t5) // get current lock value
bne t0, 20f // if ne, lock owned
10: LDP_L t0, 0(t5) // get current lock value
bis t5, zero, t3 // t3 = ownership value
bne t0, 20f // if ne, spin lock owned
STP_C t3, 0(t5) // set lock owned
beq t3, 15f // if eq, store conditional failure
mb // synchronize memory access
#endif
//
// The attempt to acquire the specified spin lock succeeded.
//
// Save the old Irql at the address saved by the caller.
// Insure that the old Irql is updated with longword granularity.
//
ldq_u t1, 0(t11) // read quadword containing KIRQL
bic t11, 3, t2 // get address of containing longword
mskbl t1, t11, t1 // clear byte position of KIRQL
bis v0, zero, a0 // save old irql
insbl v0, t11, v0 // get KIRQL to correct byte
bis t1, v0, t1 // merge KIRQL into quadword
extll t1, t2, t1 // extract containing longword
stl t1, 0(t2) // store containing longword
ldil v0, TRUE // set return value
ret zero, (ra) // return
//
// The attempt to acquire the specified spin lock failed. Lower IRQL to its
// previous value and return FALSE.
//
// N.B. The lower IRQL code is duplicated here is avoid any extra overhead
// since this is such a common operation.
//
#if !defined(NT_UP)
20: bis v0, zero, a0 // set old IRQL value
SWAP_IRQL // change back to old irql(a0)
ldil v0, FALSE // set return to failed
ret zero, (ra) // return
//
// Attempt to acquire spinlock failed.
//
15: br zero, 10b // retry spinlock
#endif
.end KeTryToAcquireSpinLock
//++
//
// KIRQL
// KiAcquireSpinLock (
// IN PKSPIN_LOCK SpinLock
// )
//
// Routine Description:
//
// This function acquires a kernel spin lock.
//
// N.B. This function assumes that the current IRQL is set properly.
//
// Arguments:
//
// SpinLock (a0) - Supplies a pointer to a kernel spin lock.
//
// Return Value:
//
// None.
//
//--
LEAF_ENTRY(KiAcquireSpinLock)
ALTERNATE_ENTRY(KeAcquireSpinLockAtDpcLevel)
#if !defined(NT_UP)
GET_CURRENT_THREAD // v0 = current thread address
10: LDP_L t2, 0(a0) // get current lock value
bis v0, zero, t3 // set ownership value
bne t2, 15f // if ne, spin lock owned
STP_C t3, 0(a0) // set spin lock owned
beq t3, 15f // if eq, store conditional failure
mb // synchronize memory access
ret zero, (ra) // return
//
// attempt to acquire spinlock failed.
//
15: LDP t2, 0(a0) // get current lock value
beq t2, 10b // retry acquire lock if unowned
br zero, 15b // loop in cache until lock free
#else
ret zero, (ra) // return
#endif
.end KiAcquireSpinLock
//++
//
// VOID
// KiReleaseSpinLock (
// IN PKSPIN_LOCK SpinLock
// )
//
// Routine Description:
//
// This function releases a kernel spin lock.
//
// N.B. This function assumes that the current IRQL is set properly.
//
// Arguments:
//
// SpinLock (a0) - Supplies a pointer to an executive spin lock.
//
// Return Value:
//
// None.
//
//--
LEAF_ENTRY(KiReleaseSpinLock)
ALTERNATE_ENTRY(KeReleaseSpinLockFromDpcLevel)
#if !defined(NT_UP)
mb // synchronize memory accesss
STP zero, 0(a0) // set spin lock not owned
#endif
ret zero, (ra) // return
.end KiReleaseSpinLock
//++
//
// KIRQL
// KiTryToAcquireSpinLock (
// IN PKSPIN_LOCK SpinLock
// )
//
// Routine Description:
//
// This function attempts to acquires the specified kernel spinlock. If
// the spinlock can be acquired, then TRUE is returned. Otherwise, FALSE
// is returned.
//
// N.B. This function assumes that the current IRQL is set properly.
//
// Arguments:
//
// SpinLock (a0) - Supplies a pointer to a kernel spin lock.
//
// Return Value:
//
// If the spin lock is acquired, then a value of TRUE is returned.
// Otherwise, a value of FALSE is returned.
//
//--
LEAF_ENTRY(KiTryToAcquireSpinLock)
#if !defined(NT_UP)
GET_CURRENT_THREAD // v0 = current thread address
10: LDP_L t2, 0(a0) // get current lock value
bis v0, zero, t3 // set ownership value
bne t2, 20f // if ne, spin lock owned
STP_C t3, 0(a0) // set spin lock owned
beq t3, 15f // if eq, conditional store failed
mb // synchronize memory access
ldil v0, TRUE // set success return value
ret zero, (ra) // return
20: ldil v0, FALSE // set failure return value
ret zero, (ra) // return
//
// Attempt to acquire spinlock failed.
//
15: br zero, 10b // retry
#else
ldil v0, TRUE // set success return value
ret zero, (ra) // return
#endif
.end KiTryToAcquireSpinLock
//++
//
// BOOLEAN
// KeTestSpinLock (
// IN PKSPIN_LOCK SpinLock
// )
//
// Routine Description:
//
// This function tests a kernel spin lock. If the spinlock is
// busy, FALSE is returned. If not, TRUE is returned. The spinlock
// is never acquired. This is provided to allow code to spin at low
// IRQL, only raising the IRQL when there is a reasonable hope of
// acquiring the lock.
//
// Arguments:
//
// SpinLock (a0) - Supplies a pointer to a kernel spin lock.
//
// Return Value:
//
// TRUE - Spinlock appears available
// FALSE - SpinLock is busy
//--
#if !defined(NT_UP)
LEAF_ENTRY(KeTestSpinLock)
LDP t0, (a0) // get current spinlock value
ldil v0, 1 // default TRUE
cmovne t0, zero, v0 // if t0 != 0, return FALSE
ret zero, (ra) // return
.end KeTestSpinLock
#endif
SBTTL("Acquire Queued SpinLock and Raise IRQL")
//++
//
// VOID
// KeAcquireInStackQueuedSpinLock (
// IN PKSPIN_LOCK SpinLock,
// IN PKLOCK_QUEUE_HANDLE LockHandle
// )
//
// VOID
// KeAcquireInStackQueuedSpinLockRaiseToSynch (
// IN PKSPIN_LOCK SpinLock,
// IN PKLOCK_QUEUE_HANDLE LockHandle
// )
//
// Routine Description:
//
// This function raises the current IRQL to either synchronization or
// dispatch level and acquires the specified queued spinlock.
//
// Arguments:
//
// SpinLock (a0) - Supplies a pointer to a spin lock.
//
// LockHandle (a1) - Supplies a pointer to a lock handle.
//
// Return Value:
//
// None.
//
//--
LEAF_ENTRY(KeAcquireInStackQueuedSpinLock)
ldil v0, DISPATCH_LEVEL // get dispatch level IRQL
br zero, 5f //
ALTERNATE_ENTRY(KeAcquireInStackQueuedSpinLockRaiseToSynch)
ldl v0, KiSynchIrql // get synch level IRQL
5: mov a1, t5 // save address of lock handle
#if !defined(NT_UP)
STP zero, LqhNext(t5) // set next link to NULL
STP a0, LqhLock(t5) // set spin lock address
mb // synchronize memory access
#endif
mov v0, a0 // set new Irql value
SWAP_IRQL // raise Irql to specified level
stl v0, LqhOldIrql(t5) // save old IRQL
#if !defined(NT_UP)
ADDP t5, LqhNext, t5 // set address of lock queue
br zero, KxqAcquireQueuedSpinLock // finish in common code
#else
ret zero, (ra) // return
#endif
.end KeAcquireInStackQueuedSpinLock
SBTTL("Acquire Queued SpinLock and Raise IRQL")
//++
//
// KIRQL
// KeAcquireQueuedSpinLock (
// IN KSPIN_LOCK_QUEUE_NUMBER Number
// )
//
// KIRQL
// KeAcquireQueuedSpinLockRaiseToSynch (
// IN KSPIN_LOCK_QUEUE_NUMBER Number
// )
//
// Routine Description:
//
// This function raises the current IRQL to synchronization level and
// acquires the specified queued spinlock.
//
// Arguments:
//
// Number (a0) - Supplies the queued spinlock number.
//
// Return Value:
//
// The previous IRQL is returned as the function value.
//
//--
LEAF_ENTRY(KeAcquireQueuedSpinLock)
addq a0, a0, t5 // account for two addresses
ldil a0, DISPATCH_LEVEL // get dispatch level IRQL
br zero, 5f //
ALTERNATE_ENTRY(KeAcquireQueuedSpinLockRaiseToSynch)
addq a0, a0, t5 // account for two addresses
ldl a0, KiSynchIrql // get synch level IRQL
5: SWAP_IRQL // raise Irql to specified level
#if !defined(NT_UP)
bis v0, zero, t0 // save previous Irql
GET_PROCESSOR_CONTROL_BLOCK_BASE // get current prcb address
SPADDP t5, v0, t5 // entry address
lda t5, PbLockQueue(t5) //
ALTERNATE_ENTRY(KxqAcquireQueuedSpinLock)
LDP t4, LqLock(t5) // get associated spinlock address
10: LDP_L t3, 0(t4) // get current lock value
bis t5, zero, t2 // set lock ownership value
STP_C t2, 0(t4) //
beq t2, 50f // if eq, conditional store failed
bne t3, 30f // if ne, lock already owned
bis t4, LOCK_QUEUE_OWNER, t4 // set lock owner bit in lock entry
STP t4, LqLock(t5) //
mb // synchronize subsequent reads
20: bis t0, zero, v0 // set old IRQL value
#endif
ret zero, (ra) // return
//
// The lock is owned by another processor. Set the lock bit in the current
// processor lock queue entry, set the next link in the previous lock queue
// entry, and spin on the current processor's lock bit.
//
#if !defined(NT_UP)
30: bis t4, LOCK_QUEUE_WAIT, t4 // set lock wait bit in lock entry
STP t4, LqLock(t5) //
mb // synchronize memory access
STP t5, LqNext(t3) // set address of lock queue entry
40: LDP t4, LqLock(t5) // get lock address and lock wait bit
blbc t4, 20b // if lbc (lock wait), ownership granted
br zero, 40b // try again
//
// Conditional store failed.
//
50: br zero, 10b // try again
#endif
.end KeAcquireQueuedSpinLock
SBTTL("Release Queued SpinLock and Lower IRQL")
//++
//
// VOID
// KeReleaseInStackQueuedSpinLock (
// IN PKLOCK_QUEUE_HANDLE LockHandle
// )
//
// Routine Description:
//
// This function releases a queued spinlock and lowers the IRQL to its
// previous value.
//
// Arguments:
//
// LockHandle (a0) - Supplies a pointer to a lock handle.
//
// Return Value:
//
// None.
//
//--
LEAF_ENTRY(KeReleaseInStackQueuedSpinLock)
ldl t0, LqhOldIrql(a0) // save old IRQL
ADDP a0, LqhNext, t5 // set address of lock queue
br zero, KxqReleaseQueuedSpinLock // finish in common code
.end KeReleaseInStackQueuedSpinLock
SBTTL("Release Queued SpinLock and Lower IRQL")
//++
//
// VOID
// KeReleaseQueuedSpinLock (
// IN KSPIN_LOCK_QUEUE_NUMBER Number,
// IN KIRQL OldIrql
// )
//
// Routine Description:
//
// This function releases a queued spinlock and lowers the IRQL to its
// previous value.
//
// Arguments:
//
// Number (a0) - Supplies the queued spinlock number.
//
// OldIrql (a1) - Supplies the previous IRQL value.
//
// Return Value:
//
// None.
//
//--
LEAF_ENTRY(KeReleaseQueuedSpinLock)
bis a1, zero, t0 // save old IRQL
#if !defined(NT_UP)
addq a0, a0, t5 // account for two addresses
GET_PROCESSOR_CONTROL_BLOCK_BASE // get current prcb address
SPADDP t5, v0, t5 // compute per processor lock queue
lda t5, PbLockQueue(t5) // entry address
#endif
ALTERNATE_ENTRY(KxqReleaseQueuedSpinLock)
#if !defined(NT_UP)
mb // synchronize memory access
LDP t4, LqLock(t5) // get associated spin lock address
bic t4, LOCK_QUEUE_OWNER, t4 // clear lock owner bit in lock entry
10: LDP_L t3, 0(t4) // get current lock ownership value
xor t3, t5, t2 // set lock ownership value
bne t2, 20f // if ne, another processor waiting
STP_C t2, 0(t4) // set new ownership value
beq t2, 10b // if eq, conditional store failed
STP t4, LqLock(t5) // store associated spin lock address
mb // synchronize memory access
#endif
bis t0, zero, a0 // set old IRQL value
SWAP_IRQL // lower IRQL to previous level
ret zero, (ra) // return
//
// Another processor has inserted its lock queue entry in the lock queue,
// but has not yet written its lock queue entry address in the current
// processor's next link. Spin until the lock queue address is written.
//
#if !defined(NT_UP)
20: LDP t3, LqNext(t5) // get next lock queue entry address
beq t3, 20b // if eq, address not written yet
//
// Grant the next process in the lock queue ownership of the spinlock.
//
LDP t2, LqLock(t3) // get spinlock address and wait bit
STP zero, LqNext(t5) // clear next lock queue entry address
STP t4, LqLock(t5) // store associated spin lock address
bic t2, LOCK_QUEUE_WAIT, t2 // clear lock wait bit in lock entry
bis t2, LOCK_QUEUE_OWNER, t2 // set lock owner bit in lock entry
STP t2, LqLock(t3) //
mb // synchronize memory access
bis t0, zero, a0 // set old IRQL value
SWAP_IRQL // lower IRQL to previous level
ret zero, (ra) // return
#endif
.end KeReleaseQueuedSpinLock
SBTTL("Try to Acquire Queued SpinLock and Raise IRQL")
//++
//
// LOGICAL
// KeTryToAcquireQueuedSpinLock (
// IN KSPIN_LOCK_QUEUE_NUMBER Number
// OUT PKIRQL OldIrql
// )
//
// LOGICAL
// KeTryToAcquireQueuedSpinLockRaiseToSynch (
// IN KSPIN_LOCK_QUEUE_NUMBER Number
// OUT PKIRQL OldIrql
// )
//
// Routine Description:
//
// This function raises the current IRQL to synchronization level and
// attempts to acquire the specified queued spinlock. If the spinlock
// cannot be acquired, then IRQL is restored and FALSE is returned as
// the function value. Otherwise, TRUE is returned as the function
// value.
//
// Arguments:
//
// Number (a0) - Supplies the queued spinlock number.
//
// OldIrql (a1) - Supplies a pointer to a variable that receives the
// the previous IRQL value.
//
// Return Value:
//
// If the spin lock is acquired, then a value of TRUE is returned.
// Otherwise, a value of FALSE is returned.
//
//--
LEAF_ENTRY(KeTryToAcquireQueuedSpinLock)
addq a0, a0, t5 // account for two addresses
ldil a0, DISPATCH_LEVEL // get dispatch level irql
br zero, 5f //
ALTERNATE_ENTRY(KeTryToAcquireQueuedSpinLockRaiseToSynch)
addq a0, a0, t5 // account for two addresses
ldl a0, KiSynchIrql // get synch level irql
5: bis a1, zero, t0 // save previous irql address
SWAP_IRQL // raise irql to specified level
bis v0, zero, t1 // save previous irql
#if !defined(NT_UP)
GET_PROCESSOR_CONTROL_BLOCK_BASE // get current prcb address
SPADDP t5, v0, t5 // compute per processor lock queue
lda t5, PbLockQueue(t5) // entry address
LDP t4, LqLock(t5) // get associated spinlock address
//
// Try to acquire the specified spinlock.
//
// N.B. A noninterlocked test is done before the interlocked attempt. This
// allows spinning without interlocked cycles.
//
LDP t3, 0(t4) // get current lock value
bne t3, 20f // if ne, lock owned
10: LDP_L t3, 0(t4) // get current lock value
bis t5, zero, t2 // set lock ownership value
bne t3, 20f // if ne, spin lock owned
STP_C t2, 0(t4) // set lock owned
beq t2, 30f // if eq, store conditional failure
//
// The attempt to acquire the specified spin lock succeeded. Set the spin
// lock owner bit and save the old irql at the address specified by the
// caller. Insure that the old Irql is updated with longword granularity.
//
bis t4, LOCK_QUEUE_OWNER, t4 // set lock owner bit in lock entry
STP t4, LqLock(t5) //
mb // synchronize memory access
#endif
ldq_u t2, 0(t0) // get quadword containing irql
bic t0, 3, t3 // get containing longword address
mskbl t2, t0, t2 // clear byte position of Irql
insbl t1, t0, t1 // shift irql to correct byte
bis t2, t1, t2 // merge irql into quadword
extll t2, t3, t2 // extract containing longword
stl t2, 0(t3) // store containing longword
ldil v0, TRUE // set return value
ret zero, (ra) // return
//
// The attempt to acquire the specified spin lock failed. Lower IRQL to its
// previous value and return FALSE.
//
#if !defined(NT_UP)
20: bis t1, zero, a0 // set old irql value
SWAP_IRQL // lower irql to previous level
ldil v0, FALSE // set return value
ret zero, (ra) // return
//
// Attempt to acquire spinlock failed.
//
30: br zero, 10b // retry spinlock
#endif
.end KeTryToAcquireQueuedSpinLock
SBTTL("Acquire Queued SpinLock at Current IRQL")
//++
// VOID
// KeAcquireInStackQueuedSpinLockAtDpcLevel (
// IN PKSPIN_LOCK SpinLock,
// IN PKLOCK_QUEUE_HANDLE LockHandle
// )
//
// Routine Description:
//
// This function acquires the specified queued spinlock at the current
// IRQL.
//
// Arguments:
//
// SpinLock (a0) - Supplies the address of a spin lock.
//
// LockHandle (a1) - Supplies the address of an in stack lock handle.
//
// Return Value:
//
// None.
//
//--
LEAF_ENTRY(KeAcquireInStackQueuedSpinLockAtDpcLevel)
#if !defined(NT_UP)
STP zero, LqhNext(a1) // set next link to NULL
STP a0, LqhLock(a1) // set spin lock address
#endif
ADDP a1, LqhNext, a0 // compute address of lock queue
//++
//
// VOID
// KeAcquireQueuedSpinLockAtDpcLevel (
// IN PKSPIN_LOCK_QUEUE LockQueue
// )
//
// Routine Description:
//
// This function acquires the specified queued spinlock at the current
// IRQL.
//
// Arguments:
//
// LockQueue (a0) - Supplies the address of the lock queue entry.
//
// Return Value:
//
// None.
//
//--
ALTERNATE_ENTRY(KeAcquireQueuedSpinLockAtDpcLevel)
#if !defined(NT_UP)
mb // synchronize memory access
LDP t4, LqLock(a0) // get associated spinlock address
10: LDP_L t3, 0(t4) // get current lock value
bis a0, zero, t2 // set lock ownership value
STP_C t2, 0(t4) //
beq t2, 50f // if eq, conditional store failed
bne t3, 30f // if ne, lock already owned
bis t4, LOCK_QUEUE_OWNER, t4 // set lock owner bit in lock entry
STP t4, LqLock(a0) //
mb // synchronize subsequent reads
#endif
20: ret zero, (ra) // return
//
// The lock is owned by another processor. Set the lock bit in the current
// processor lock queue entry, set the next link in the previous lock queue
// entry, and spin on the current processor's lock bit.
//
#if !defined(NT_UP)
30: bis t4, LOCK_QUEUE_WAIT, t4 // set lock wait bit in lock entry
STP t4, LqLock(a0) //
mb // synchronize memory access
STP a0, LqNext(t3) // set address of lock queue entry
40: LDP t4, LqLock(a0) // get lock address and lock wait bit
blbc t4, 20b // if lbc (lock wait), ownership granted
br zero, 40b // try again
//
// Conditional store failed.
//
50: br zero, 10b // try again
#endif
.end KeAcquireInStackQueuedSpinLockAtDpcLevel
SBTTL("Release Queued SpinLock at Current IRQL")
//++
//
// VOID
// KeReleaseInStackQueuedSpinLockFromDpcLevel (
// IN PKLOCK_QUEUE_HANDLE LockHandle
// )
//
// Routine Description:
//
// This function releases a queued spinlock and preserves the current
// IRQL.
//
// Arguments:
//
// LockHandle (a0) - Supplies the address of a lock handle.
//
// Return Value:
//
// None.
//
//--
LEAF_ENTRY(KeReleaseInStackQueuedSpinLockFromDpcLevel)
ADDP a0, LqhNext, a0 // compute address of lock queue
//++
//
// VOID
// KeReleaseQueuedSpinLockFromDpcLevel (
// IN PKSPIN_LOCK_QUEUE LockQueue
// )
//
// Routine Description:
//
// This function releases a queued spinlock and preserves the current
// IRQL.
//
// Arguments:
//
// LockQueue (a0) - Supplies the address of the lock queue entry.
//
// Return Value:
//
// None.
//
//--
ALTERNATE_ENTRY(KeReleaseQueuedSpinLockFromDpcLevel)
#if !defined(NT_UP)
mb // synchronize memory access
LDP t4, LqLock(a0) // get associate spin lock address
bic t4, LOCK_QUEUE_OWNER, t4 // clear lock owner bit in lock entry
10: LDP_L t3, 0(t4) // get current lock ownership value
xor t3, a0, t2 // set lock ownership value
bne t2, 20f // if ne, another processor waiting
STP_C t2, 0(t4) // set new ownership value
beq t2, 10b // if eq, conditional store failed
STP t4, LqLock(a0) //
mb // synchronize memory access
ret zero, (ra) // return
//
// Another processor has inserted its lock queue entry in the lock queue,
// but has not yet written its lock queue entry address in the current
// processor's next link. Spin until the lock queue address is written.
//
20: LDP t3, LqNext(a0) // get next lock queue entry address
beq t3, 20b // if eq, address not written yet
//
// Grant the next process in the lock queue ownership of the spinlock.
//
LDP t2, LqLock(t3) // get spinlock address and lock bit
STP zero, LqNext(a0) // clear next lock queue entry address
STP t4, LqLock(a0) //
bic t2, LOCK_QUEUE_WAIT, t2 // clear lock wait bit in lock entry
bis t2, LOCK_QUEUE_OWNER, t2 // set lock owner bit in lock entry
STP t2, LqLock(t3) //
mb // synchronize memory access
#endif
ret zero, (ra) // return
.end KeReleaseInStackQueuedSpinLockFromDpcLevel
Reference in a new issue Copy permalink