554 lines
14 KiB
ArmAsm
554 lines
14 KiB
ArmAsm
//++
|
||
//
|
||
// Copyright (c) 1989 Microsoft Corporation
|
||
//
|
||
// Module Name:
|
||
//
|
||
// critsect.s
|
||
//
|
||
// Abstract:
|
||
//
|
||
// This module implements functions to support user mode critical sections.
|
||
//
|
||
// Author:
|
||
//
|
||
// William K. Cheung (wcheung) 18-Sep-95
|
||
//
|
||
// Revision History:
|
||
//
|
||
// 07-Jul-97 bl Updated to EAS2.3
|
||
//
|
||
// 08-Feb-96 Updated to EAS2.1
|
||
//
|
||
//--
|
||
|
||
#include "ksia64.h"
|
||
|
||
.file "critsect.s"
|
||
|
||
//
|
||
// intra-module functions to be called.
|
||
//
|
||
|
||
PublicFunction(RtlpWaitForCriticalSection)
|
||
PublicFunction(RtlpUnWaitCriticalSection)
|
||
|
||
|
||
//++
|
||
//
|
||
// NTSTATUS
|
||
// RtlEnterCriticalSection(
|
||
// IN PRTL_CRITICAL_SECTION CriticalSection
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function enters a critical section.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// CriticalSection (a0) - supplies a pointer to a critical section.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// STATUS_SUCCESS or raises an exception if an error occured.
|
||
//
|
||
// Algorithm in C:
|
||
//
|
||
// NTSTATUS
|
||
// RtlEnterCriticalSection(
|
||
// IN PRTL_CRITICAL_SECTION CriticalSection
|
||
// )
|
||
// {
|
||
// PTEB Teb;
|
||
// LONG OriginalValue;
|
||
// HANDLE CurrentThreadId;
|
||
// DWORD SpinCount;
|
||
//
|
||
// Teb = NtCurrentTeb();
|
||
// CurrentThreadId = Teb->ClientId.UniqueThread;
|
||
// SpinCount = CriticalSection->SpinCount;
|
||
//
|
||
// if (SpinCount == 0) {
|
||
// nospin:
|
||
// OriginalValue = AtomicIncrement(CriticalSection->LockCount);
|
||
//
|
||
// if (OriginalValue != -1) {
|
||
// if (CriticalSection->OwningThread != CurrentThreadId) {
|
||
// RtlpWaitForCriticalSection(CriticalSection);
|
||
// CriticalSection->OwningThread = CurrentThreadId;
|
||
// CriticalSection->RecursionCount = 0;
|
||
// } else {
|
||
// CriticalSection->RecursionCount++;
|
||
// }
|
||
// } else {
|
||
// CriticalSection->OwningThread = CurrentThreadId;
|
||
// CriticalSection->RecursionCount = 0;
|
||
// }
|
||
//
|
||
// return STATUS_SUCCESS;
|
||
//
|
||
// } else { /* spin count not 0 */
|
||
//
|
||
// if (CriticalSection->OwningThread == CurrentThread) {
|
||
// AtomicIncrement(CriticalSection->LockCount);
|
||
// CriticalSection->RecursionCount++;
|
||
// return STATUS_SUCCESS;
|
||
// } else {
|
||
// do {
|
||
// if (!CmpXchg(CriticalSection->LockCount,0,-1)) {
|
||
// Lock acquired
|
||
// CriticalSection->OwningThread = CurrentThreadId;
|
||
// CriticalSection->RecursionCount = 0;
|
||
// return STATUS_SUCCESS;
|
||
// }
|
||
// if (CriticalSection->LockCount >= 1) goto nospin;
|
||
//
|
||
// while (CriticalSection->LockCount == 0) {
|
||
// if (!--SpinCount) goto nospin;
|
||
// }
|
||
// } while (1) // CriticalSection->LockCount == -1, not owned
|
||
// }
|
||
// }
|
||
// }
|
||
//
|
||
//--
|
||
|
||
|
||
|
||
NESTED_ENTRY(RtlEnterCriticalSection)
|
||
|
||
//
|
||
// register aliases
|
||
//
|
||
|
||
rpThreadId=t3
|
||
rOldLockCount=t4
|
||
rpLockCount=t5
|
||
rRecursionCount=t6
|
||
rOwnerId=t7
|
||
rpSpinCount=t9
|
||
rSpinCount=t10
|
||
rT1=t11
|
||
rT2=t12
|
||
|
||
pSpin=pt0
|
||
pNoSpin=pt1
|
||
pNotOwn=pt2
|
||
pNotAcq=pt3
|
||
pFree=pt5
|
||
pHeld=pt6
|
||
pGo=pt7
|
||
pWait=pt8
|
||
|
||
//
|
||
// alloc regs and save brp & pfs
|
||
//
|
||
|
||
NESTED_SETUP(1,5,1,0)
|
||
add rpLockCount = CsLockCount, a0
|
||
;;
|
||
|
||
//
|
||
// more register aliases
|
||
//
|
||
|
||
rThreadId = loc2
|
||
rpOwner = loc3
|
||
rpCsRecursion = loc4
|
||
|
||
PROLOGUE_END
|
||
|
||
//
|
||
// Swizzle pointers to address the RecursionCount and
|
||
// LockCount fields in the critical section structure.
|
||
//
|
||
|
||
lfetch.nt1 [rpLockCount]
|
||
nop.f 0
|
||
add rpSpinCount = CsSpinCount, a0
|
||
|
||
add rpCsRecursion = CsRecursionCount, a0
|
||
nop.f 0
|
||
add rpThreadId = TeClientId+CidUniqueThread, teb
|
||
;;
|
||
|
||
//
|
||
// Load the id of the currently running thread, anticipating
|
||
// that it may be needed.
|
||
//
|
||
|
||
ld8 rThreadId = [rpThreadId]
|
||
ld4 rSpinCount = [rpSpinCount]
|
||
add rpOwner = CsOwningThread, a0
|
||
;;
|
||
|
||
//
|
||
// Branch out if spin count is non-zero
|
||
//
|
||
cmp4.ne pSpin, pNoSpin = rSpinCount, zero
|
||
mov v0 = STATUS_SUCCESS
|
||
(pSpin) br.spnt RecsSpin
|
||
|
||
//
|
||
// Atomically increment the lock count at location (rpLockCount)
|
||
// and put the old value in register "rOldLockCount".
|
||
// Swizzle a pointer to address the thread id field in teb structure.
|
||
//
|
||
|
||
RecsNoSpin:
|
||
fetchadd4.acq rOldLockCount = [rpLockCount], 1
|
||
ld4.nt1 rRecursionCount = [rpCsRecursion]
|
||
;;
|
||
|
||
//
|
||
// Check the original value of lock count to determine if the
|
||
// lock is free. If the value is -1, it is free.
|
||
//
|
||
|
||
cmp4.eq pFree, pHeld = -1, rOldLockCount
|
||
;;
|
||
|
||
//
|
||
// if lock is not free, get the thread id of its current owner
|
||
// otherwise, save the currently running thread's id.
|
||
//
|
||
|
||
(pHeld) ld8 rOwnerId = [rpOwner]
|
||
(pFree) st8 [rpOwner] = rThreadId
|
||
(pFree) st4 [rpCsRecursion] = zero
|
||
(pFree) br.ret.sptk.clr brp // return
|
||
;;
|
||
|
||
//
|
||
// if lock is not free, compare the owner id of the critical section against
|
||
// that of the thread to determine if this thread is the owner.
|
||
// otherwise, return to caller.
|
||
//
|
||
|
||
cmp.eq pGo, pWait = rThreadId, rOwnerId
|
||
mov out0 = a0
|
||
add rRecursionCount = 1, rRecursionCount
|
||
;;
|
||
|
||
//
|
||
// if the thread has already owned the lock, save the updated
|
||
// recursion count and return.
|
||
// otherwise, wait for the critical section to be released.
|
||
//
|
||
|
||
(pGo) st4 [rpCsRecursion] = rRecursionCount
|
||
(pGo) br.ret.sptk brp
|
||
(pWait) br.call.spnt.many brp = RtlpWaitForCriticalSection
|
||
;;
|
||
|
||
st8.rel [rpOwner] = rThreadId
|
||
st4 [rpCsRecursion] = zero
|
||
mov v0 = STATUS_SUCCESS
|
||
|
||
NESTED_RETURN
|
||
|
||
// A nonzero spin count is specified
|
||
//
|
||
|
||
RecsSpin:
|
||
|
||
ld8 rOwnerId = [rpOwner]
|
||
mov rT1 = -1
|
||
;;
|
||
zxt4 rT1 = rT1 // zero extend for compare with
|
||
;; // 4 byte lock value
|
||
|
||
mov ar.ccv = rT1 // compare value
|
||
cmp.ne pNotOwn = rOwnerId, rThreadId
|
||
(pNotOwn) br.spnt RecsNotOwn
|
||
|
||
//
|
||
// The critical section is owned by the current thread. Increment the lock
|
||
// count and the recursion count.
|
||
//
|
||
|
||
ld4 rRecursionCount = [rpCsRecursion]
|
||
fetchadd4.acq rOldLockCount = [rpLockCount], 1
|
||
;;
|
||
add rRecursionCount = 1, rRecursionCount
|
||
;;
|
||
|
||
st4 [rpCsRecursion] = rRecursionCount
|
||
br.ret.sptk.clr brp // return
|
||
|
||
//
|
||
// A nonzero spin count is specified and the current thread is not the owner.
|
||
//
|
||
|
||
RecsNotOwn:
|
||
cmpxchg4.acq rT1 = [rpLockCount], zero // try to acquire lock
|
||
;;
|
||
|
||
cmp4.ne pNotAcq = -1, rT1
|
||
(pNotAcq) br.spnt RecsNotAcq
|
||
|
||
|
||
//
|
||
// The critical section has been acquired. Set the owning thread and the initial
|
||
// recursion count and return success.
|
||
//
|
||
|
||
st8 [rpOwner] = rThreadId
|
||
st4 [rpCsRecursion] = zero
|
||
br.ret.sptk.clr brp // return
|
||
|
||
//
|
||
// The critical section is currently owned. Spin until it is either unowned
|
||
// or the spin count has reached zero.
|
||
//
|
||
// If LockCount > 0, then there are waiters. Don't spin because
|
||
// the lock will not free.
|
||
//
|
||
|
||
RecsNotAcq:
|
||
ld4 rOldLockCount = [rpLockCount]
|
||
;;
|
||
|
||
cmp4.eq pNotOwn = -1, rOldLockCount
|
||
cmp4.gt pNoSpin = rOldLockCount, zero
|
||
(pNoSpin) br.spnt RecsNoSpin
|
||
(pNotOwn) br.spnt RecsNotOwn
|
||
|
||
add rSpinCount = -1, rSpinCount
|
||
;;
|
||
|
||
cmp4.eq pNoSpin, pSpin = zero, rSpinCount
|
||
(pNoSpin) br.spnt RecsNoSpin
|
||
(pSpin) br.sptk RecsNotAcq
|
||
;;
|
||
|
||
NESTED_EXIT(RtlEnterCriticalSection)
|
||
|
||
//++
|
||
//
|
||
// NTSTATUS
|
||
// RtlLeaveCriticalSection(
|
||
// IN PRTL_CRITICAL_SECTION CriticalSection
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function enters a critical section.
|
||
//
|
||
// N.B. This function is duplicated in the runtime library.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// CriticalSection (a0) - Supplies a pointer to a critical section.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// STATUS_SUCCESS is returned as the function value.
|
||
//
|
||
// Algorithm in C:
|
||
//
|
||
// NTSTATUS
|
||
// RtlLeaveCriticalSection(
|
||
// IN PRTL_CRITICAL_SECTION CriticalSection
|
||
// )
|
||
// {
|
||
// LONG NewRecursionCount;
|
||
// LONG OldLockCount;
|
||
// BOOL ToRelease;
|
||
//
|
||
// ASSERT(CriticalSection->RecursionCount >= 0)
|
||
//
|
||
// if (CriticalSection->RecursionCount != 0) {
|
||
// CriticalSection->RecursionCount -= 1;
|
||
// AtomicDecrement(CriticalSection->LockCount);
|
||
// return STATUS_SUCCESS;
|
||
// }
|
||
//
|
||
// CriticalSection->OwningThread = 0;
|
||
// OldLockCount = AtomicDecrement(CriticalSection->LockCount);
|
||
//
|
||
// if ( OldLockCount != 0 ) {
|
||
// RtlpUnWaitCriticalSection(CriticalSection);
|
||
// }
|
||
//
|
||
// return STATUS_SUCCESS;
|
||
// }
|
||
//
|
||
//--
|
||
|
||
//
|
||
// register aliases
|
||
//
|
||
|
||
|
||
NESTED_ENTRY(RtlLeaveCriticalSection)
|
||
|
||
rpOwner=t0
|
||
rOldLockCount=t1
|
||
rRecursionCount=t2
|
||
rpLockCount=t5
|
||
rpCsRecursion=t9
|
||
|
||
pHold=pt0
|
||
pRel=pt1
|
||
pGo=pt7
|
||
pWait=pt8
|
||
|
||
NESTED_SETUP(1,2,1,0)
|
||
add rpCsRecursion = CsRecursionCount, a0
|
||
;;
|
||
|
||
PROLOGUE_END
|
||
|
||
//
|
||
// load recursion count
|
||
// swizzle pointers to address the LockCount and OwningThread
|
||
// fields in the critical section structure.
|
||
//
|
||
|
||
ld4 rRecursionCount = [rpCsRecursion]
|
||
add rpOwner = CsOwningThread, a0
|
||
add rpLockCount = CsLockCount, a0
|
||
;;
|
||
|
||
//
|
||
// check if the original value of the recursion count to determine
|
||
// if the lock is to be released.
|
||
//
|
||
// decrement the register copy of recursion count by 1 and save
|
||
// the new value in temp register
|
||
//
|
||
|
||
cmp.ne pHold, pRel = zero, rRecursionCount
|
||
add rRecursionCount = -1, rRecursionCount
|
||
add v0 = STATUS_SUCCESS, zero // return STATUS_SUCCESS
|
||
;;
|
||
|
||
//
|
||
// save the updated recursion count into the critical section structure.
|
||
//
|
||
// atomically decrement the lock count.
|
||
//
|
||
// if lock is still held, return to caller.
|
||
//
|
||
// Note: An atomic fetch & add with release form is used here
|
||
// all previous memory accesses are visible at this point.
|
||
//
|
||
// Note: All updates to the Critical Section structure MUST be complete
|
||
// prior to the lock count being decremented which releases the
|
||
// lock.
|
||
//
|
||
|
||
(pHold) st4 [rpCsRecursion] = rRecursionCount
|
||
(pRel) st8 [rpOwner] = zero // clear the owner field
|
||
;;
|
||
fetchadd4.rel rOldLockCount = [rpLockCount], -1
|
||
(pHold) br.ret.sptk.clr brp // return to caller
|
||
;;
|
||
|
||
//
|
||
// The lock is now free, check the original value of the lock count to
|
||
// determine if any other thread is waiting for this critical section.
|
||
// If no thread is waiting, return to caller immediately.
|
||
//
|
||
|
||
cmp4.ge pGo, pWait = zero, rOldLockCount
|
||
(pGo) br.ret.sptk.clr brp // return to caller
|
||
|
||
mov out0 = a0
|
||
(pWait) br.call.spnt.many brp = RtlpUnWaitCriticalSection
|
||
|
||
mov v0 = STATUS_SUCCESS // return STATUS_SUCCESS
|
||
|
||
NESTED_RETURN
|
||
;;
|
||
|
||
NESTED_EXIT(RtlLeaveCriticalSection)
|
||
|
||
|
||
//++
|
||
//
|
||
// BOOL
|
||
// RtlTryEnterCriticalSection(
|
||
// IN PRTL_CRITICAL_SECTION CriticalSection
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function attempts to enter a critical section without blocking.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// CriticalSection (a0) - Supplies a pointer to a critical section.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// If the critical section was successfully entered, then a value of TRUE
|
||
// is returned as the function value. Otherwise, a value of FALSE is returned
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(RtlTryEnterCriticalSection)
|
||
|
||
//
|
||
// register aliases
|
||
//
|
||
|
||
rT0=t0
|
||
rT1=t1
|
||
rpThreadId=t3
|
||
rOldLockCount=t4
|
||
rpLockCount=t5
|
||
rRecursionCount=t6
|
||
rOwnerId=t7
|
||
rpCsRecursion=t8
|
||
rThreadId=t9
|
||
rpOwner=t10
|
||
|
||
pFree=pt5
|
||
pHeld=pt6
|
||
pOwn=pt7
|
||
pFail=pt8
|
||
|
||
alloc rT0 = ar.pfs, 1, 0, 0, 0
|
||
movl rT1 = 0xffffffff
|
||
;;
|
||
|
||
mov ar.ccv = rT1
|
||
add rpOwner = CsOwningThread, a0
|
||
add rpLockCount = CsLockCount, a0
|
||
;;
|
||
|
||
cmpxchg4.acq rOldLockCount = [rpLockCount], r0, ar.ccv
|
||
add rpCsRecursion = CsRecursionCount, a0
|
||
add rpThreadId = TeClientId+CidUniqueThread, teb
|
||
;;
|
||
|
||
ld8 rOwnerId = [rpOwner]
|
||
cmp4.eq pFree, pHeld = rT1, rOldLockCount
|
||
;;
|
||
|
||
|
||
ld8.nt1 rThreadId = [rpThreadId]
|
||
(pHeld) ld4.nt1 rRecursionCount = [rpCsRecursion]
|
||
mov v0 = TRUE
|
||
;;
|
||
|
||
(pFree) st4 [rpCsRecursion] = zero
|
||
(pFree) st8 [rpOwner] = rThreadId
|
||
(pHeld) cmp.eq pOwn, pFail = rThreadId, rOwnerId
|
||
(pFree) br.ret.sptk.clr brp
|
||
;;
|
||
|
||
(pOwn) fetchadd4.acq rT0 = [rpLockCount], 1
|
||
(pOwn) add rRecursionCount = 1, rRecursionCount
|
||
nop.i 0
|
||
;;
|
||
|
||
(pOwn) st4.rel [rpCsRecursion] = rRecursionCount
|
||
(pFail) mov v0 = FALSE
|
||
br.ret.sptk.clr brp
|
||
|
||
LEAF_EXIT(RtlTryEnterCriticalSection)
|