windows-nt/Source/XPSP1/NT/base/hals/halacpi/amd64/mpipi.c

/*++

Copyright (c) 2000 Microsoft Corporation

Module Name:

    mpipi.c

Abstract:

    This module provides the HAL support for interprocessor interrupts and
    processor initialization for MPS systems.

Author:

    Forrest Foltz (forrestf) 27-Oct-2000

Environment:

    Kernel mode only.

Revision History:

--*/

#include "halcmn.h"

#define HAL_FORCEINLINE __forceinline

//
// External functions
//

VOID
HalpResetThisProcessor (
    VOID
    );

ULONG
DetectAcpiMP (
    OUT PBOOLEAN IsConfiguredMp,
    IN  PLOADER_PARAMETER_BLOCK LoaderBlock
    );

//
// External data
//

extern KAFFINITY HalpNodeAffinity[];
extern INTERRUPT_DEST HalpIpiDestinationMap[sizeof(KAFFINITY)][256];
extern BOOLEAN HalpStaticIntAffinity;
extern UCHAR rgzBadHal[];

//
// Local types
//

typedef
VOID
(*HAL_GENERIC_IPI_FUNCTION) (
    ULONG_PTR Context
    );

//
// Local prototypes
//

VOID
HalInitApicInterruptHandlers(
    VOID
    );

//
// External data
//

extern UCHAR HalpPICINTToVector[16];

//
// Local data and defines
//

KSPIN_LOCK HalpBroadcastLock;
KAFFINITY volatile HalpBroadcastTargets;
ULONG_PTR HalpBroadcastContext;
HAL_GENERIC_IPI_FUNCTION HalpBroadcastFunction;
PKPCR HalpProcessorPCR[MAXIMUM_PROCESSORS];

//
// HalpGlobal8259Mask is used to avoid reading the PIC to get the current
// interrupt mask; format is the same as for SET_8259_MASK, i.i.,
// bits 7:0 -> PIC1, 15:8 -> PIC2
//

USHORT HalpGlobal8259Mask = 0;

#define GENERIC_IPI (DELIVER_FIXED | LOGICAL_DESTINATION | ICR_USE_DEST_FIELD | APIC_GENERIC_VECTOR)
#define APIC_IPI    (DELIVER_FIXED | LOGICAL_DESTINATION | ICR_USE_DEST_FIELD | APIC_IPI_VECTOR)

//
// Globals and constants used to log local apic errors
//

#define LogApicErrors TRUE
#if LogApicErrors

//
// Structure defining the layout of an apic error record.
// 

typedef struct _APIC_ERROR {
    union {
        struct {
            UCHAR SendChecksum:1;
            UCHAR ReceiveChecksum:1;
            UCHAR SendAccept:1;
            UCHAR ReceiveAccept:1;
            UCHAR Reserved1:1;
            UCHAR SendVector:1;
            UCHAR ReceiveVector:1;
            UCHAR RegisterAddress:1;
        };
        UCHAR AsByte;
    };
    UCHAR Processor;
} APIC_ERROR, *PAPIC_ERROR;

#define APIC_ERROR_LOG_SIZE 128

//
// Count of local apic errors.
// 

ULONG HalpLocalApicErrorCount = 0;

//
// Apic error log.  This is circular, indexed by
// HalpLocalApicErrorCount % APIC_ERROR_LOG_SIZE.
//

APIC_ERROR HalpApicErrorLog[APIC_ERROR_LOG_SIZE];

//
// Spinlock used to protect access to HalpLocalApicErrorCount.
//

KSPIN_LOCK HalpLocalApicErrorLock;

#endif


HAL_FORCEINLINE
VOID
HalpSendIpiWorker (
    IN UCHAR TargetSet,
    IN ULONG Command
    )

/*++

Routine Description:

    This routine is called to send an IPI command to a set of processors
    on a single node.

Parameters:

    TargetSet - Specifies the processor identifiers within the node.

    Command - Specifies the IPI command to send.

Return Value:

    None.

--*/

{
    ULONG destination;

    //
    // Only high byte of the destination is used.  Wait until the Apic is
    // not busy before sending.  Continue without waiting, there will be
    // another wait after all IPIs have been submitted.
    // 

    destination = (ULONG)TargetSet << DESTINATION_SHIFT;

    HalpStallWhileApicBusy();
    LOCAL_APIC(LU_INT_CMD_HIGH) = destination;
    LOCAL_APIC(LU_INT_CMD_LOW) = Command;
}


HAL_FORCEINLINE
VOID
HalpSendNodeIpi (
    IN KAFFINITY Affinity,
    IN ULONG Command
    )

/*++

Routine Description:

Parameters:

    Affinity - Specifies the set of processors to receive the IPI.

    Command - Specifies the IPI command to send.

Return Value:

    None.

--*/

{
    KAFFINITY remainingProcessors;
    PKAFFINITY nodeAffinity;
    ULONG chunkNo;
    INTERRUPT_DEST intDest;
    INTERRUPT_DEST intDestSum;
    UCHAR affinityChunk;

    //
    // Declare a local union that can be used to access an affinity
    // both chunk-wise and as a whole.
    //

    union {
        UCHAR Chunks[sizeof(KAFFINITY)];
        KAFFINITY Whole;
    } affinity;

    //
    // Affinity has some number of target processors indicated.  Each
    // target processor is a member of a cluster of processors, or "node".
    //
    // For each node, determine whether it contains any of the target
    // processors.  If so, send an IPI command targeting those processors
    // and send it to the node.
    //

    nodeAffinity = HalpNodeAffinity;
    remainingProcessors = Affinity & HalpActiveProcessors;

    while (remainingProcessors != 0) {

        //
        // Iterate through the node affinities here until a node containing
        // at least some of the targeted processors is encountered.
        // 

        do {

            //
            // Determine the set of target CPUs that can be found in this
            // node.  
            // 
    
            affinity.Whole = *nodeAffinity & remainingProcessors;
    
            //
            // Point nodeAffinity at the affinity for the next cluster.
            //
    
            nodeAffinity += 1;
            ASSERT((nodeAffinity - HalpNodeAffinity) < MAX_NODES);

        } while (affinity.Whole == 0);

        //
        // Remove the processors that will be processed on this node from
        // the set of processors remaining to be processed.
        //

        remainingProcessors ^= affinity.Whole;

        //
        // Accumulate the logical target mask 
        //

        intDestSum.LogicalId = 0;
        chunkNo = 0;
        do {

            //
            // Isolate a chunk of the processor affinity mask.
            //

            affinityChunk = affinity.Chunks[chunkNo];

            //
            // Use that chunk as an index into HalpIpiDestinationMap[][],
            // retrieving the logical sum of all node/processor IDs that
            // are associated with each bit that is set in that affinity
            // chunk.
            //

            intDest = HalpIpiDestinationMap[chunkNo][affinityChunk];

            //
            // Now, sum it with the logical ID mask that is being accumulated
            // for each affinity chunk.
            //

            intDestSum.LogicalId |= intDest.LogicalId;

            //
            // Indicate that the processors represented in this chunk
            // have been processed.  When there are no more processors
            // left to process for this node, send the IPI and proceed
            // to the next node.
            //

            affinity.Chunks[chunkNo] = 0;
            chunkNo += 1;

        } while (affinity.Whole != 0);

        //
        // intDest contains an accumulated set of hardware processor
        // identifiers, representing all of the processors on this node that
        // should receive the command.
        // 

        HalpSendIpiWorker(intDestSum.LogicalId,Command);
    }
}


VOID
HalpSendIpi (
    IN KAFFINITY Affinity,
    IN ULONG Command
    )

/*++

Routine Description:

    Affinity - Specifies the set of processors to receive the IPI.

    Command - Specifies the IPI command to send.

Parameters:

    None.

Return Value:

    None.

--*/

{
    ULONG flags;

    //
    // Disable interrupts and call the appropriate routine.
    // 

    flags = HalpDisableInterrupts();
    if (HalpMaxProcsPerCluster == 0) {

        //
        // We know that the maximum number of processors is 8,
        // so send the IPI directly.
        //

        ASSERT((Affinity & 0xFF) == Affinity);
        HalpSendIpiWorker((UCHAR)Affinity,Command);

    } else {

        //
        // Send an IPI to one or mode nodes.
        //

        HalpSendNodeIpi(Affinity,Command);
    }

    //
    // Stall until the last IPI has been sent, restore interrupts and
    // return.
    //

    HalpStallWhileApicBusy();
    HalpRestoreInterrupts(flags);
}

VOID
HalInitializeProcessor(
    ULONG ProcessorNumber,
    PLOADER_PARAMETER_BLOCK LoaderBlock
    )

/*++

Routine Description:

    Initialize hal pcr values for current processor (if any)
    (called shortly after processor reaches kernel, before
    HalInitSystem if P0)

    IPI's and KeReadir/LowerIrq's must be available once this function
    returns.  (IPI's are only used once two or more processors are
    available)

   . Enable IPI interrupt (makes sense for P1, P2, ...).
   . Save Processor Number in PCR.
   . if (P0)
       . determine if the system is a PC+MP,
       . if not a PC+MP System Halt;
   . Enable IPI's on CPU.

Arguments:

    Number - Logical processor number of calling processor

Return Value:

    None.

--*/

{
    PKPCR pcr;
    KAFFINITY affinity;
    KAFFINITY oldAffinity;
    ULONG detectAcpiResult;
    BOOLEAN isMp;

    affinity = (KAFFINITY)1 << ProcessorNumber;
    pcr = KeGetPcr();

    //
    // Mark all interrupts as disabled, and store the processor number and
    // the default stall scale factor in the pcr.
    //

    pcr->Idr = 0xFFFFFFFF;
    pcr->Number = (UCHAR)ProcessorNumber;
    pcr->StallScaleFactor = INITIAL_STALL_COUNT;

    //
    // Record the pcr pointer in our lookup table and set the affinity
    // bit in our set of active processors.
    // 

    HalpProcessorPCR[ProcessorNumber] = pcr;
    HalpActiveProcessors |= affinity;

    if (HalpStaticIntAffinity == 0) {

        //
        // Interrupts can go to any processor
        //

        HalpDefaultInterruptAffinity |= affinity;

    } else {

        //
        // Interrupts go only to the highest numbered processor
        //

        if (HalpDefaultInterruptAffinity < affinity) {
            HalpDefaultInterruptAffinity = affinity;
        }
    }

    if (ProcessorNumber == 0) {

        KeInitializeSpinLock(&HalpBroadcastLock);

#if LogApicErrors

        KeInitializeSpinLock(&HalpLocalApicErrorLock);

#endif

        //
        // Determine whether the system we are on is an MPS system.
        //
        // DetectMPS has a parameter we don't currently use.  It's a boolean
        // which is set to TRUE if the system we're on is an MP system.
        // We could have a UP MPS system.
        //
        // The DetectMPS routine also allocates virtual addresses for all of
        // the APICs in the system.
        //

#if defined(ACPI_HAL)
        detectAcpiResult = DetectAcpiMP(&isMp,LoaderBlock);
#else
        detectAcpiResult = DetectMPS(&isMp);
#endif
        if (detectAcpiResult == FALSE) {
            HalDisplayString(rgzBadHal);

            HalpDisableInterrupts();
            while (TRUE) {
                HalpHalt();
            }
        }

        HalpRegisterKdSupportFunctions(LoaderBlock);

        //
        // Mask all PIC interrupts
        //

        HalpGlobal8259Mask = 0xFFFF;
        SET_8259_MASK(HalpGlobal8259Mask);
    }

    //
    // All processors execute this code
    //

    HalInitApicInterruptHandlers();
    HalpInitializeLocalUnit();
}

VOID
HalInitApicInterruptHandlers(
    VOID
    )

/*++

Routine Description:

    This routine installs the interrupt vector in the IDT for the APIC
    spurious interrupt.

Arguments:

    None.

Return Value:

    None.

--*/

{
    PKPCR pcr;
    PKIDTENTRY64 idt;

    KiSetHandlerAddressToIDTIrql(PIC1_SPURIOUS_VECTOR,
                                 PicSpuriousService37,
                                 NULL,
                                 0);

    KiSetHandlerAddressToIDTIrql(APIC_SPURIOUS_VECTOR,
                                 HalpApicSpuriousService,
                                 NULL,
                                 0);
}

__forceinline
VOID
HalpPollForBroadcast (
    VOID
    )

/*++

Routine Description:

    Checks whether the current processor has a broadcast function pending
    and, if so, clears it's pending bit and calls the function.

Arguments:

    None.

Return Value:

    None.

--*/

{
    KAFFINITY affinity;
    ULONG_PTR broadcastContext;
    HAL_GENERIC_IPI_FUNCTION broadcastFunction;
    KAFFINITY broadcastTargets;

    affinity = KeGetPcr()->CurrentPrcb->SetMember;
    if ((HalpBroadcastTargets & affinity) != 0) {

        //
        // A pending generic IPI call appears to be pending for this
        // processor.  Pick up the function pointer and context locally.
        //

        broadcastFunction = HalpBroadcastFunction;
        broadcastContext = HalpBroadcastContext;

        //
        // Atomically acknowledge the broadcast.  If the broadcast is still
        // pending for this processor, then call it.
        //

        // BUGBUG
        // broadcastTargets = InterlockedAnd64(&HalpBroadcastTargets,~affinity);
        broadcastTargets &= ~affinity;
        if ((broadcastTargets & affinity) != 0) {
            broadcastFunction(broadcastContext);
        }
    }
}

VOID
HalpGenericCall(
    IN HAL_GENERIC_IPI_FUNCTION BroadcastFunction,
    IN ULONG Context,
    IN KAFFINITY TargetProcessors
    )

/*++

Routine Description:

    Causes the WorkerFunction to be called on the specified target
    processors.  The WorkerFunction is called at CLOCK2_LEVEL-1
    (Must be below IPI_LEVEL in order to prevent system deadlocks).

Enviroment:

    Must be called with interrupts enabled.
    Must be called with IRQL = CLOCK2_LEVEL-1

--*/

{
    //
    // Nothing to do if no target processors have been specified.
    //

    if (TargetProcessors == 0) {
        return;
    }

    //
    // Acquire the broadcast lock, polling for broadcasts while spinning.
    //

    while (KeTryToAcquireSpinLockAtDpcLevel(&HalpBroadcastLock) == FALSE) {
        do {
            HalpPollForBroadcast();
        } while (KeTestSpinLock(&HalpBroadcastLock) == FALSE);
    }

    //
    // We own the broadcast lock.  Store the broadcast parameters
    // into the broadcast prameters and send the generic IPI.
    //

    HalpBroadcastFunction = BroadcastFunction;
    HalpBroadcastContext = Context;
    HalpBroadcastTargets = TargetProcessors;
    HalpSendIpi(TargetProcessors,GENERIC_IPI);

    //
    // Wait for all processors to pick up the IPI and process the generic
    // call, then release the broadcast lock.
    // 

    do {
        HalpPollForBroadcast();
    } while (HalpBroadcastTargets != 0);

    KeReleaseSpinLockFromDpcLevel(&HalpBroadcastLock);
}


ULONG
HalpWaitForPending (
    IN ULONG Count,
    IN ULONG volatile *ICR
    )

/*++

Routine Description:

    Spins waiting for the DELIVERY_PENDING bit in the ICR to clear or
    until spinning Count times.

Arguments:

    Count - Number of times through the loop before giving up.

    ICR - Pointer to the ICR register containing the DELIVERY_PENDING
          status bit.

Return Value:

    Zero if the DELIVERY_PENDING bit has cleared within the number of
    test cycles, non-zero otherwise.

--*/

{
    ULONG countRemaining;

    countRemaining = Count;
    while (countRemaining > 0) {

        if ((*ICR & DELIVERY_PENDING) != 0) {
            break;
        }
        countRemaining -= 1;
    }

    return countRemaining;
}

VOID
HalRequestIpi (
    IN KAFFINITY Affinity
    )

/*++

Routine Description:

    Requests an interprocessor interrupt

Arguments:

    Affinity - Supplies the set of processors to be interrupted

Return Value:

    None.

--*/

{
    HalpSendIpi(Affinity,APIC_IPI);
}

BOOLEAN
HalpApicRebootService (
    IN PKINTERRUPT Interrupt,
    IN PVOID ServiceContext
    )

/*++

Routine Description:

    This is the ISR that handles Reboot interrupts.

Arguments:

    Interrupt - Supplies a pointer to the kernel interrupt object

    ServiceContext - Supplies the service context

Return Value:

    None.  This routine does not return.

--*/

{
    UNREFERENCED_PARAMETER(Interrupt);
    UNREFERENCED_PARAMETER(ServiceContext);

    LOCAL_APIC(LU_TPR) = APIC_REBOOT_VECTOR;

    //
    // EOI the local APIC.  Warm reset does not reset the 82489 APIC
    // so if we don't EOI here we'll never see an interrupt after the
    // reboot.
    //

    LOCAL_APIC(LU_EOI) = 0;

    //
    // Reset this processor.  This function will not return.
    // 

    HalpResetThisProcessor();
    ASSERT(FALSE);

    return TRUE;
}


BOOLEAN
HalpBroadcastCallService (
    IN PKINTERRUPT Interrupt,
    IN PVOID ServiceContext
    )

/*++

Routine Description:

    This is the ISR that handles broadcast call interrupts.

Arguments:

    Interrupt - Supplies a pointer to the kernel interrupt object

    ServiceContext - Supplies the service context

Return Value:

    TRUE

--*/

{
    UNREFERENCED_PARAMETER(Interrupt);
    UNREFERENCED_PARAMETER(ServiceContext);

    HalpPollForBroadcast();
    return TRUE;
}

BOOLEAN
HalpIpiHandler (
    IN PKINTERRUPT Interrupt,
    IN PVOID ServiceContext
    )

/*++

Routine Description:

    This routine is entered as the result of an interrupt generated by
    interprocessor communication.

Arguments:

    Interrupt - Supplies a pointer to the kernel interrupt object

    ServiceContext - Supplies the service context

Return Value:

    TRUE

--*/

{
    UNREFERENCED_PARAMETER(Interrupt);

    KiIpiServiceRoutine(Interrupt->TrapFrame,NULL);

    return TRUE;
}

BOOLEAN
HalpLocalApicErrorService (
    IN PKINTERRUPT Interrupt,
    IN PVOID ServiceContext
    )

/*++

Routine Description:

    This routine is entered as the result of an interrupt generated by
    a local apic error.  It clears the error and, if apic error logging
    is turned on, records information about the error.

Arguments:

    Interrupt - Supplies a pointer to the kernel interrupt object

    ServiceContext - Supplies the service context

Return Value:

    TRUE

--*/

{
    ULONG flags;
    PAPIC_ERROR apicError;
    ULONG index;
    ULONG errorStatus;
    PKPCR pcr;

#if LogApicErrors

    //
    // Take the apic error log lock, get a pointer to the next available
    // error log slot, and increment the error count.
    //

    flags = HalpAcquireHighLevelLock(&HalpLocalApicErrorLock);

    index = HalpLocalApicErrorCount % APIC_ERROR_LOG_SIZE;
    apicError = &HalpApicErrorLog[index];
    HalpLocalApicErrorCount += 1;

#endif

    //
    // The Apic EDS (Rev 4.0) says you have to write before you read.
    // This doesn't work.  The write clears the status bits, but the P6 works
    // according to the EDS.
    //
    // For AMD64, for now assume that things work according to the EDS spec.
    //

    LOCAL_APIC(LU_ERROR_STATUS) = 0;
    errorStatus = LOCAL_APIC(LU_ERROR_STATUS);

#if LogApicErrors

    //
    // Fill in the error log and release the apic error log lock.
    //

    pcr = KeGetPcr();
    apicError->AsByte = (UCHAR)errorStatus;
    apicError->Processor = pcr->Number;

    HalpReleaseHighLevelLock(&HalpLocalApicErrorLock,flags);

#endif

    return TRUE;
}


BOOLEAN
PicNopHandlerInt (
    IN PKINTERRUPT Interrupt,
    IN PVOID Context
    )

/*++

Routine Description:

    This handler is designed to be installed on a system to field any PIC
    interrupts when there are not supposed to be any delivered.

    This routine EOIs the PIC and returns.

Arguments:

    Interrupt - Supplies a pointer to the kernel interrupt object

    ServiceContext - Supplies the service context

Return Value:

    TRUE

--*/

{
    UCHAR irq;

    AMD64_COVERAGE_TRAP();

    //
    // Context is the PIC IRQ
    //

    ASSERT((ULONG_PTR)Context <= 15);

    irq = (UCHAR)(ULONG_PTR)(Context);
    if (irq <= 7) {

        WRITE_PORT_UCHAR(PIC1_PORT0,irq | OCW2_SPECIFIC_EOI);

    } else {

        if (irq == 0x0D) {
            WRITE_PORT_UCHAR(I386_80387_BUSY_PORT, 0);
        }

        WRITE_PORT_UCHAR(PIC2_PORT0,OCW2_NON_SPECIFIC_EOI);
        WRITE_PORT_UCHAR(PIC1_PORT0,OCW2_SPECIFIC_EOI | PIC_SLAVE_IRQ);
    }

    return TRUE;
}


BOOLEAN
PicInterruptHandlerInt (
    IN PKINTERRUPT Interrupt,
    IN PVOID Context
    )

/*++

Routine Description:

    These handlers receive interrupts from the PIC and reissues them via a
    vector at the proper priority level.  This is used to provide a symetric
    interrupt distribution on a non symetric system.
 
    The PIC interrupts will normally only be received (in the PC+MP Hal) via
    an interrupt input from on either the IO Unit or the Local unit which has
    been programed as EXTINT.  EXTINT interrupts are received outside of the
    APIC priority structure (the PIC provides the vector).  We use the APIC
    ICR to generate interrupts to the proper handler at the proper priority.
 
    The EXTINT interrupts are directed to a single processor, currently P0.
    There is no good reason why they can't be directed to another processor.
 
    Since one processor must absorb the overhead of redistributing PIC
    interrupts the interrupt handling on a system using EXTINT interrupts is
    not symetric.
 
Arguments:

    Interrupt - Supplies a pointer to the kernel interrupt object

    ServiceContext - Supplies the service context

Return Value:

    TRUE

--*/

{
    UCHAR irq;
    UCHAR isrRegister;
    UCHAR ipiVector;

    AMD64_COVERAGE_TRAP();

    //
    // Context is the PIC IRQ
    //

    ASSERT((ULONG_PTR)Context <= 15);

    irq = (UCHAR)(ULONG_PTR)(Context);
    if (irq == 7) {

        //
        // Check to see if this is a spurious interrupt
        //

        WRITE_PORT_UCHAR(PIC1_PORT0,OCW3_READ_ISR);
        IO_DELAY();
        isrRegister = READ_PORT_UCHAR(PIC1_PORT0);
        if ((isrRegister & 0x80) == 0) {

            //
            // Spurious.
            //

            return TRUE;
        }
    }

    if (irq == 0x0D) {

        WRITE_PORT_UCHAR(I386_80387_BUSY_PORT,0);

    } else if (irq == 0x1F) {

        WRITE_PORT_UCHAR(PIC2_PORT0,OCW3_READ_ISR);
        IO_DELAY();
        isrRegister = READ_PORT_UCHAR(PIC2_PORT0);
        if ((isrRegister & 0x80) == 0) {

            //
            // Spurious.
            //

            return TRUE;
        }
    }

    if (irq <= 7) {

        //
        // Master PIC
        //

        WRITE_PORT_UCHAR(PIC1_PORT0,irq | OCW2_SPECIFIC_EOI);

    } else {

        //
        // Slave PIC
        //

        WRITE_PORT_UCHAR(PIC2_PORT0,OCW2_NON_SPECIFIC_EOI);
        WRITE_PORT_UCHAR(PIC1_PORT0,OCW2_SPECIFIC_EOI | PIC_SLAVE_IRQ);
    }

    ipiVector = HalpPICINTToVector[irq];

    if (ipiVector != 0) {

        HalpStallWhileApicBusy();
        if (irq == 8) {

            //
            // Clock interrupt
            //

            LOCAL_APIC(LU_INT_CMD_LOW) =
                DELIVER_FIXED | ICR_SELF | APIC_CLOCK_VECTOR;

        } else {

            //
            // Write the IPI command to the Memory Mapped Register
            //

            LOCAL_APIC(LU_INT_CMD_HIGH) = DESTINATION_ALL_CPUS;
            LOCAL_APIC(LU_INT_CMD_LOW) = ipiVector;
        }
    }

    return TRUE;
}