able/windows-nt
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
windows-nt/Source/XPSP1/NT/base/wow64/mscpu/fraglib/cpu/patchfn.c

2638 lines
72 KiB
C
Raw Normal View History

Add source files
2020-09-26 03:20:57 -05:00
/*++
Copyright (c) 1996-2000 Microsoft Corporation
Module Name:
patchfn.c
Abstract:
This module contains generic functions to patch fragments. Structures
that describe the fragments to be patched. The structures live in
the processor specific directory.
Author:
Dave Hastings (daveh) creation-date 24-Jun-1995
Revision History:
Barry Bond (barrybo) 1-Apr-1995
Switch the PPC build to the AXP model of patching
24-Aug-1999 [askhalid] copied from 32-bit wx86 directory and make work for 64bit.
20-Sept-1999[barrybo] added FRAG2REF(LockCmpXchg8bFrag32, ULONGLONG)
Notes:
--*/
#include <nt.h>
#include <ntrtl.h>
#include <nturtl.h>
#include <windows.h>
#define _WX86CPUAPI_
#include "wx86.h"
#include "wx86nt.h"
#include "wx86cpu.h"
#include "cpuassrt.h"
#include "instr.h"
#include "config.h"
#include "fragp.h"
#include "entrypt.h"
#include "compiler.h"
#include "ctrltrns.h"
#include "threadst.h"
#include "instr.h"
#include "frag.h"
#include "ptchstrc.h"
#include "mrsw.h"
#include "tc.h"
#include "codeseq.h"
#include "codesize.h"
#include "opt.h"
#if _ALPHA_
#define _codegen_
#include "soalpha.h"
#undef fTCUnlocked // this is a field in CPUCONTEXT
ULONG
GetCurrentECU(
PULONG CodeLocation
);
#endif
ASSERTNAME;
extern CHAR CallJxxHelper[];
extern CHAR CallJmpDirectHelper[];
extern CHAR IndirectControlTransferHelper[];
extern CHAR IndirectControlTransferFarHelper[];
extern CHAR CallDirectHelper[];
extern CHAR CallDirectHelper2[];
extern CHAR CallIndirectHelper[];
extern CHAR JumpToNextCompilationUnitHelper[];
#define OFFSET(type, field) ((LONG)(ULONGLONG)(&((type *)0)->field))
ULONG RegisterOffset[] = {
OFFSET(THREADSTATE, GpRegs[GP_EAX].i4), // EAX
OFFSET(THREADSTATE, GpRegs[GP_ECX].i4), // ECX
OFFSET(THREADSTATE, GpRegs[GP_EDX].i4), // EDX
OFFSET(THREADSTATE, GpRegs[GP_EBX].i4), // EBX
OFFSET(THREADSTATE, GpRegs[GP_ESP].i4), // ESP
OFFSET(THREADSTATE, GpRegs[GP_EBP].i4), // EBP
OFFSET(THREADSTATE, GpRegs[GP_ESI].i4), // ESI
OFFSET(THREADSTATE, GpRegs[GP_EDI].i4), // EDI
OFFSET(THREADSTATE, GpRegs[REG_ES]), // ES
OFFSET(THREADSTATE, GpRegs[REG_CS]), // CS
OFFSET(THREADSTATE, GpRegs[REG_SS]), // SS
OFFSET(THREADSTATE, GpRegs[REG_DS]), // DS
OFFSET(THREADSTATE, GpRegs[REG_FS]), // FS
OFFSET(THREADSTATE, GpRegs[REG_GS]), // GS
OFFSET(THREADSTATE, GpRegs[GP_EAX].i2), // AX
OFFSET(THREADSTATE, GpRegs[GP_ECX].i2), // CX
OFFSET(THREADSTATE, GpRegs[GP_EDX].i2), // DX
OFFSET(THREADSTATE, GpRegs[GP_EBX].i2), // BX
OFFSET(THREADSTATE, GpRegs[GP_ESP].i2), // SP
OFFSET(THREADSTATE, GpRegs[GP_EBP].i2), // BP
OFFSET(THREADSTATE, GpRegs[GP_ESI].i2), // SI
OFFSET(THREADSTATE, GpRegs[GP_EDI].i2), // DI
OFFSET(THREADSTATE, GpRegs[GP_EAX].i1), // AL
OFFSET(THREADSTATE, GpRegs[GP_ECX].i1), // CL
OFFSET(THREADSTATE, GpRegs[GP_EDX].i1), // DL
OFFSET(THREADSTATE, GpRegs[GP_EBX].i1), // BL
OFFSET(THREADSTATE, GpRegs[GP_EAX].hb), // AH
OFFSET(THREADSTATE, GpRegs[GP_ECX].hb), // CH
OFFSET(THREADSTATE, GpRegs[GP_EDX].hb), // DH
OFFSET(THREADSTATE, GpRegs[GP_EBX].hb) // BH
};
ULONG
PatchJumpToNextCompilationUnit(
IN PULONG PatchAddr,
IN ULONG IntelDest
)
/*++
Routine Description:
This routine ends basic blocks when the native address of the next
basic block is known.
Arguments:
PatchAddr -- address of JumpToNextCompilationUnit code in the TC
IntelDest -- intel address of the next basic block
Return Value:
Native address to jump to to resume execution.
--*/
{
DWORD TCTimestamp;
ULONG NativeDest;
ULONG NativeSize;
DECLARE_CPU;
if (cpu->flag_tf) {
return (ULONG)(ULONGLONG)&EndTranslatedCode;
}
//
// Switch from being a TC reader to a writer
//
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
//
// Compile the next basic block, and get the TC write lock
//
NativeDest = (ULONG)(ULONGLONG)NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
if (TCTimestamp == TranslationCacheTimestamp) {
PULONG CodeLocation = PatchAddr - JumpToNextCompilationUnit_SIZE;
//
// The Translation Cache wasn't flushed - replace the
// JumpToNextCompilationUnit fragment by JumpToNextCompilationUnit2
//
NativeSize=GenJumpToNextCompilationUnit2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
NativeDest,
0);
NtFlushInstructionCache(
NtCurrentProcess(),
CodeLocation,
NativeSize
);
} else {
TCTimestamp = TranslationCacheTimestamp;
if (cpu->CSTimestamp != TCTimestamp) {
//
// The cache was flushed by another thread in the small window
// between mrsw calls in this thread, we plan on jumping directly
// to NativeDest, so the CPU callstack needs to be flushed.
// Normally this would be done in the CpuSimulate() loop as a
// result of jumping to EndTranslatedCode.
//
FlushCallstack(cpu);
}
}
//
// Switch back to being a TC reader
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
if (TCTimestamp == TranslationCacheTimestamp) {
//
// TC was not flushed - NativeDest is valid
//
return NativeDest;
} else {
//
// TC was flushed while becomming a TC reader again - NativeDest
// is now bogus.
//
return (ULONG)(ULONGLONG)&EndTranslatedCode;
}
}
ULONG
PlaceJxx(
IN PULONG CodeLocation,
#if _ALPHA_
IN ULONG CurrentECU,
#endif
IN PINSTRUCTION Instruction
)
/*++
Routine Description:
This routine copies the fragment into place, and modifies the
instructions that load the destination into the register
Arguments:
Instruction - Supplies a description of the instruction the fragment
represents
CodeLocation - Supplies the address the code for the fragment has been
copied to
Return Value:
Size of code placed at CodeLocation
--*/
{
DWORD IntelDest;
PPLACEOPERATIONFN pfn;
PENTRYPOINT EP;
ULONG NativeSize;
//
// Generate the code to determine if the branch is taken or not
//
pfn = (PPLACEOPERATIONFN)FragmentArray[Instruction->Operation];
NativeSize = (*pfn)(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
CodeLocation += NativeSize/sizeof(ULONG);
// Let's see if we can place the patched version immediately
// ASSUME: The first argument is always a NOCODEGEN
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
IntelDest = Instruction->Operand1.Immed;
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
if (EP == NULL) {
//
// knowing NativeDest requires compilation. Just place the unpatched
// version to be patched later.
//
NativeSize += GenJxxBody(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
} else {
//
// We can place the patched version right away!
//
NativeSize += GenJxxBody2(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)IntelDest,
(ULONG)(ULONGLONG)EP);
}
return NativeSize;
}
ULONG
PlaceJxxSlow(
IN PULONG CodeLocation,
#if _ALPHA_
IN ULONG CurrentECU,
#endif
IN PINSTRUCTION Instruction
)
/*++
Routine Description:
This routine copies the fragment into place, and modifies the
instructions that load the destination into the register
Arguments:
Instruction - Supplies a description of the instruction the fragment
represents
CodeLocation - Supplies the address the code for the fragment has been
copied to
Return Value:
Size of code placed at CodeLocation
--*/
{
DWORD IntelDest;
PPLACEOPERATIONFN pfn;
PENTRYPOINT EP;
ULONG NativeSize;
//
// Generate the code to load RegEip with the branch-not-taken value
//
NativeSize = GenJxxStartSlow(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
//
// Generate the code to determine if the branch is taken or not
//
pfn = (PPLACEOPERATIONFN)FragmentArray[Instruction->Operation];
NativeSize += (*pfn)(CodeLocation+NativeSize/sizeof(ULONG),
#if _ALPHA_
CurrentECU,
#endif
Instruction);
CodeLocation += NativeSize/sizeof(ULONG);
// Let's see if we can place the patched version immediately
// ASSUME: The first argument is always a NOCODEGEN
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
IntelDest = Instruction->Operand1.Immed;
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
if (EP == NULL) {
//
// knowing NativeDest requires compilation. Just place the unpatched
// version to be patched later.
//
NativeSize += GenJxxBodySlow(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
} else {
//
// We can place the patched version right away!
//
NativeSize += GenJxxBodySlow2(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)IntelDest,
(ULONG)(ULONGLONG)EP);
}
return NativeSize;
}
ULONG
PlaceJxxFwd(
IN PULONG CodeLocation,
#if _ALPHA_
IN ULONG CurrentECU,
#endif
IN PINSTRUCTION Instruction
)
/*++
Routine Description:
This routine copies the fragment into place, and modifies the
instructions that load the destination into the register
Arguments:
Instruction - Supplies a description of the instruction the fragment
represents
CodeLocation - Supplies the address the code for the fragment has been
copied to
Return Value:
Size of code placed at CodeLocation
--*/
{
DWORD IntelDest;
PPLACEOPERATIONFN pfn;
PENTRYPOINT EP;
ULONG NativeSize;
//
// Generate the code to determine if the branch is taken or not
//
pfn = (PPLACEOPERATIONFN)FragmentArray[Instruction->Operation];
NativeSize = (*pfn)(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
CodeLocation += NativeSize/sizeof(ULONG);
// Let's see if we can place the patched version immediately
// ASSUME: The first argument is always a NOCODEGEN
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
IntelDest = Instruction->Operand1.Immed;
// Assert that the branch is going forward.
CPUASSERT(IntelDest > Instruction->IntelAddress);
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
if (EP == NULL) {
//
// knowing NativeDest requires compilation. Just place the unpatched
// version to be patched later.
NativeSize += GenJxxBodyFwd(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
} else {
//
// We can place the patched version right away!
//
NativeSize += GenJxxBodyFwd2(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)(ULONGLONG)EP,
0);
}
return NativeSize;
}
ULONG
PatchJxx(
IN ULONG IntelDest,
IN PULONG PatchAddr
)
/*++
Routine Description:
This routine replaces a JXXSTRUC by a JXXSTRC2 at runtime. It is called
when the conditional branch is taken, and the native address of the
destination is not yet known.
Arguments:
inteldest -- Intel destination address if the branch is taken
patchaddr -- address of the JXXSTRUC in the Translation Cache
Return Value:
Native address to jump to in order to resume execution.
--*/
{
ULONG NativeDest; // branch-taken address
PULONG fragaddr; // address of START of the fragment
DWORD TCTimestamp; // old timestamp of the Translation Cache
ULONG NativeSize;
DECLARE_CPU;
if (cpu->flag_tf) {
return (ULONG)(ULONGLONG)&EndTranslatedCode;
}
//
// Switch over to being a TC writer
//
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
//
// Get the native destination address of the branch and get the TC
// write lock
//
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
if (TCTimestamp == TranslationCacheTimestamp) {
PULONG CodeLocation = PatchAddr - CallJxx_PATCHRA_OFFSET;
//
// The Translation Cache was not flushed while switching to a TC
// writer or by the compilation. Replace JxxBody by the faster
// JxxBody2
//
NativeSize=GenJxxBody2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
(DWORD)IntelDest,
(DWORD)NativeDest);
NtFlushInstructionCache(
NtCurrentProcess(),
CodeLocation,
NativeSize
);
} else {
TCTimestamp = TranslationCacheTimestamp;
if (cpu->CSTimestamp != TCTimestamp) {
//
// The cache was flushed by another thread in the small window
// between mrsw calls in this thread, we plan on jumping directly
// to NativeDest, so the CPU callstack needs to be flushed.
// Normally this would be done in the CpuSimulate() loop as a
// result of jumping to EndTranslatedCode.
//
FlushCallstack(cpu);
}
}
//
// Switch back to being a TC reader again
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
if (TCTimestamp == TranslationCacheTimestamp) {
//
// TC was not flushed while becomming a reader again
//
return NativeDest;
} else {
//
// TC was flushed while becomming a reader. nativedest is invalid
// so do an EndTranslatedCode instead.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
ULONG
PatchJxxSlow(
IN ULONG IntelDest,
IN PULONG PatchAddr
)
/*++
Routine Description:
This routine replaces a JXXSTRUC by a JXXSTRC2 at runtime. It is called
when the conditional branch is taken, and the native address of the
destination is not yet known.
Arguments:
inteldest -- Intel destination address if the branch is taken
patchaddr -- address of the JXXSTRUC in the Translation Cache
Return Value:
Native address to jump to in order to resume execution.
--*/
{
ULONG NativeDest; // branch-taken address
PULONG fragaddr; // address of START of the fragment
DWORD TCTimestamp; // old timestamp of the Translation Cache
ULONG NativeSize;
DECLARE_CPU;
if (cpu->flag_tf) {
return (ULONG)(ULONGLONG)&EndTranslatedCode;
}
//
// Switch over to being a TC writer
//
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
//
// Get the native destination address of the branch and get the TC
// write lock
//
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
if (TCTimestamp == TranslationCacheTimestamp) {
PULONG CodeLocation = PatchAddr - CallJxxSlow_PATCHRA_OFFSET;
//
// The Translation Cache was not flushed while switching to a TC
// writer or by the compilation. Replace JxxBody by the faster
// JxxBody2
//
NativeSize = GenJxxBodySlow2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
(DWORD)IntelDest,
(DWORD)NativeDest);
NtFlushInstructionCache(
NtCurrentProcess(),
CodeLocation,
NativeSize
);
} else {
TCTimestamp = TranslationCacheTimestamp;
if (cpu->CSTimestamp != TCTimestamp) {
//
// The cache was flushed by another thread in the small window
// between mrsw calls in this thread, we plan on jumping directly
// to NativeDest, so the CPU callstack needs to be flushed.
// Normally this would be done in the CpuSimulate() loop as a
// result of jumping to EndTranslatedCode.
//
FlushCallstack(cpu);
}
}
//
// Switch back to being a TC reader again
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
if (TCTimestamp == TranslationCacheTimestamp) {
//
// TC was not flushed while becomming a reader again
//
return NativeDest;
} else {
//
// TC was flushed while becomming a reader. nativedest is invalid
// so do an EndTranslatedCode instead.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
ULONG
PatchJxxFwd(
IN ULONG IntelDest,
IN PULONG PatchAddr
)
/*++
Routine Description:
This routine replaces a JXXBODYFWD by a JXXBODYFWD2 at runtime. It is
called when the conditional branch is taken, and the native address of the
destination is not yet known.
Arguments:
inteldest -- Intel destination address if the branch is taken
patchaddr -- address of the JXXSTRUCFWD in the Translation Cache
Return Value:
Native address to jump to in order to resume execution.
--*/
{
ULONG NativeDest; // branch-taken address
PULONG fragaddr; // address of START of the fragment
DWORD TCTimestamp; // old timestamp of the Translation Cache
ULONG NativeSize;
DECLARE_CPU;
if (cpu->flag_tf) {
return (ULONG)(ULONGLONG)&EndTranslatedCode;
}
//
// Switch over to being a TC writer
//
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
//
// Get the native destination address of the branch and get the TC write lock
//
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
if (TCTimestamp == TranslationCacheTimestamp) {
PULONG CodeLocation = PatchAddr - CallJxxFwd_PATCHRA_OFFSET;
//
// The Translation Cache was not flushed while switching to a TC
// writer or by the compilation. Replace JxxBody by the faster
// JxxBodyFwd2
//
NativeSize = GenJxxBodyFwd2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
(DWORD)NativeDest,
0);
NtFlushInstructionCache(
NtCurrentProcess(),
CodeLocation,
NativeSize
);
} else {
TCTimestamp = TranslationCacheTimestamp;
if (cpu->CSTimestamp != TCTimestamp) {
//
// The cache was flushed by another thread in the small window
// between mrsw calls in this thread, we plan on jumping directly
// to NativeDest, so the CPU callstack needs to be flushed.
// Normally this would be done in the CpuSimulate() loop as a
// result of jumping to EndTranslatedCode.
//
FlushCallstack(cpu);
}
}
//
// Switch back to being a TC reader again
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
if (TCTimestamp == TranslationCacheTimestamp) {
//
// TC was not flushed while becomming a reader again
//
return NativeDest;
} else {
//
// TC was flushed while becomming a reader. nativedest is invalid
// so do an EndTranslatedCode instead.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
ULONG
PlaceJmpDirect(
IN PULONG CodeLocation,
#if _ALPHA_
IN ULONG CurrentECU,
#endif
IN PINSTRUCTION Instruction
)
/*++
Routine Description:
This routine copies the unconditional jump fragment into place and patches
the instructions that jump to EndTranslatedCode
Arguments:
Instruction - Supplies a description of the instruction the fragment
represents
CodeLocation - Supplies the address the code for the fragment has been
copied to
Return Value:
Size of code placed at CodeLocation
--*/
{
DWORD IntelDest;
PENTRYPOINT EP;
ULONG NativeSize;
// ASSUME: The first argument is always an immediate
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
IntelDest = Instruction->Operand1.Immed;
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
if (EP == NULL) {
//
// Knowing NativeDest requires compilation. Just place the unpatched version for
// now and patch it later if necessary
//
NativeSize = GenCallJmpDirect(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
} else {
//
// We can place the patched version right away!
//
NativeSize = GenCallJmpDirect2(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)(ULONGLONG) EP,
IntelDest);
}
return NativeSize;
}
ULONG
PatchJmpDirect(
IN PULONG PatchAddr,
IN ULONG IntelDest
)
/*++
Routine Description:
This routine patches a JMPDIRECT to a JMPDIRECT2. It is called when
the native destination address of a jmp instruction is not yet known.
It patches the jmp to jump directly to the corresponding native code.
Arguments:
PatchAddr -- address of the JMPDIRECT in the Translation Cache
IntelDest -- intel address of the destination of the jmp
Return Value:
Native address to jump to in order to resume execution
--*/
{
DWORD TCTimestamp;
ULONG NativeDest;
ULONG NativeSize;
DECLARE_CPU;
if (cpu->flag_tf) {
return (ULONG)(ULONGLONG)&EndTranslatedCode;
}
//
// Switch from being a TC reader to a writer
//
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
//
// Compile the destination of the jmp and get the TC write lock
//
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
if (TCTimestamp == TranslationCacheTimestamp) {
PULONG CodeLocation = PatchAddr - CallJmpDirect_PATCHRA_OFFSET;
//
// The Translation Cache wasn't flushed - replace the JMPDIRECT
// fragment by JMPDIRECT2
//
NativeSize = GenCallJmpDirect2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
(ULONG)NativeDest,
IntelDest);
NtFlushInstructionCache(
NtCurrentProcess(),
(PVOID)CodeLocation,
NativeSize
);
} else {
TCTimestamp = TranslationCacheTimestamp;
if (cpu->CSTimestamp != TCTimestamp) {
//
// The cache was flushed by another thread in the small window
// between mrsw calls in this thread, we plan on jumping directly
// to NativeDest, so the CPU callstack needs to be flushed.
// Normally this would be done in the CpuSimulate() loop as a
// result of jumping to EndTranslatedCode.
//
FlushCallstack(cpu);
}
}
//
// Switch back to being a TC reader
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
if (TCTimestamp == TranslationCacheTimestamp) {
//
// TC was not flushed - nativedest is valid
//
return NativeDest;
} else {
//
// TC was flushed while becomming a TC reader again - nativedest
// is now bogus.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
ULONG
PlaceJmpDirectSlow(
IN PULONG CodeLocation,
#if _ALPHA_
IN ULONG CurrentECU,
#endif
IN PINSTRUCTION Instruction
)
/*++
Routine Description:
This routine copies the unconditional jump fragment into place and patches
the instructions that jump to EndTranslatedCode
Arguments:
Instruction - Supplies a description of the instruction the fragment
represents
CodeLocation - Supplies the address the code for the fragment has been
copied to
Return Value:
Size of code placed at CodeLocation
--*/
{
DWORD IntelDest;
PENTRYPOINT EP;
ULONG NativeSize;
// ASSUME: The first argument is always an immediate
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
IntelDest = Instruction->Operand1.Immed;
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
if (EP == NULL) {
//
// Knowing NativeDest requires compilation. Just place the unpatched version for
// now and patch it later if necessary
//
NativeSize = GenCallJmpDirectSlow(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
} else {
//
// We can place the patched version right away!
//
NativeSize = GenCallJmpDirectSlow2(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)(ULONGLONG) EP,
IntelDest);
}
return NativeSize;
}
ULONG
PatchJmpDirectSlow(
IN PULONG PatchAddr,
IN ULONG IntelDest
)
/*++
Routine Description:
This routine patches a JMPDIRECT to a JMPDIRECT2. It is called when
the native destination address of a jmp instruction is not yet known.
It patches the jmp to jump directly to the corresponding native code.
Arguments:
PatchAddr -- address of the JMPDIRECT in the Translation Cache
IntelDest -- intel address of the destination of the jmp
Return Value:
Native address to jump to in order to resume execution
--*/
{
DWORD TCTimestamp;
ULONG NativeDest;
ULONG NativeSize;
DECLARE_CPU;
if (cpu->flag_tf) {
return (ULONG)(ULONGLONG)&EndTranslatedCode;
}
//
// Switch from being a TC reader to a writer
//
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
//
// Compile the destination of the jmp and get the TC write lock
//
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
if (TCTimestamp == TranslationCacheTimestamp) {
PULONG CodeLocation = PatchAddr - CallJmpDirectSlow_PATCHRA_OFFSET;
//
// The Translation Cache wasn't flushed - replace the JMPDIRECT
// fragment by JMPDIRECT2
//
NativeSize = GenCallJmpDirectSlow2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
(ULONG)NativeDest,
IntelDest);
NtFlushInstructionCache(
NtCurrentProcess(),
(PVOID)CodeLocation,
NativeSize
);
} else {
TCTimestamp = TranslationCacheTimestamp;
if (cpu->CSTimestamp != TCTimestamp) {
//
// The cache was flushed by another thread in the small window
// between mrsw calls in this thread, we plan on jumping directly
// to NativeDest, so the CPU callstack needs to be flushed.
// Normally this would be done in the CpuSimulate() loop as a
// result of jumping to EndTranslatedCode.
//
FlushCallstack(cpu);
}
}
//
// Switch back to being a TC reader
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
if (TCTimestamp == TranslationCacheTimestamp) {
//
// TC was not flushed - nativedest is valid
//
return NativeDest;
} else {
//
// TC was flushed while becomming a TC reader again - nativedest
// is now bogus.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
ULONG
PlaceJmpFwdDirect(
IN PULONG CodeLocation,
#if _ALPHA_
IN ULONG CurrentECU,
#endif
IN PINSTRUCTION Instruction
)
/*++
Routine Description:
This routine copies the unconditional jump fragment into place and patches
the instructions that jump to EndTranslatedCode
Arguments:
Instruction - Supplies a description of the instruction the fragment
represents
CodeLocation - Supplies the address the code for the fragment has been
copied to
Return Value:
Size of code placed at CodeLocation
--*/
{
DWORD IntelDest;
PENTRYPOINT EP;
ULONG NativeSize;
// ASSUME: The first argument is always an immediate
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
IntelDest = Instruction->Operand1.Immed;
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
if (EP == NULL) {
//
// Knowing NativeDest requires compilation. Just place the unpatched version for
// now and patch it later if necessary
//
NativeSize = GenCallJmpFwdDirect(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
} else {
//
// We can place the patched version right away!
//
NativeSize = GenCallJmpFwdDirect2(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)(ULONGLONG) EP,
0);
}
return NativeSize;
}
ULONG
PatchJmpFwdDirect(
IN PULONG PatchAddr,
IN ULONG IntelDest
)
/*++
Routine Description:
This routine patches a JMPDIRECT to a JMPDIRECT2. It is called when
the native destination address of a jmp instruction is not yet known.
It patches the jmp to jump directly to the corresponding native code.
Arguments:
PatchAddr -- address of the JMPDIRECT in the Translation Cache
IntelDest -- intel address of the destination of the jmp
Return Value:
Native address to jump to in order to resume execution
--*/
{
DWORD TCTimestamp;
ULONG NativeDest;
ULONG NativeSize;
DECLARE_CPU;
if (cpu->flag_tf) {
return (ULONG)(ULONGLONG)&EndTranslatedCode;
}
//
// Switch from being a TC reader to a writer
//
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
//
// Compile the destination of the jmp and get the TC write lock
//
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
if (TCTimestamp == TranslationCacheTimestamp) {
PULONG CodeLocation = PatchAddr - CallJmpFwdDirect_PATCHRA_OFFSET;
//
// The Translation Cache wasn't flushed - replace the JMPDIRECT
// fragment by JMPDIRECT2
//
NativeSize = GenCallJmpFwdDirect2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
(ULONG)NativeDest,
0);
NtFlushInstructionCache(
NtCurrentProcess(),
(PVOID)CodeLocation,
NativeSize
);
} else {
TCTimestamp = TranslationCacheTimestamp;
if (cpu->CSTimestamp != TCTimestamp) {
//
// The cache was flushed by another thread in the small window
// between mrsw calls in this thread, we plan on jumping directly
// to NativeDest, so the CPU callstack needs to be flushed.
// Normally this would be done in the CpuSimulate() loop as a
// result of jumping to EndTranslatedCode.
//
FlushCallstack(cpu);
}
}
//
// Switch back to being a TC reader
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
if (TCTimestamp == TranslationCacheTimestamp) {
//
// TC was not flushed - nativedest is valid
//
return NativeDest;
} else {
//
// TC was flushed while becomming a TC reader again - nativedest
// is now bogus.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
ULONG
PlaceJmpfDirect(
IN PULONG CodeLocation,
#if _ALPHA_
IN ULONG CurrentECU,
#endif
IN PINSTRUCTION Instruction
)
/*++
Routine Description:
This routine copies the unconditional jump fragment into place and patches
the instructions that jump to EndTranslatedCode
Arguments:
Instruction - Supplies a description of the instruction the fragment
represents
CodeLocation - Supplies the address the code for the fragment has been
copied to
Return Value:
Size of code placed at CodeLocation
--*/
{
DWORD IntelDest;
PENTRYPOINT EP;
ULONG NativeSize;
// ASSUME: The first argument is always an IMM, pointing at the address
CPUASSERT( Instruction->Operand1.Type == OPND_IMM );
IntelDest = Instruction->Operand1.Immed;
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)*(UNALIGNED DWORD *)IntelDest);
if (EP == NULL) {
//
// Knowing NativeDest requires compilation. Just place the unpatched
// version for now and patch it later if necessary
//
NativeSize = GenCallJmpfDirect(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
} else {
//
// We can place the patched version right away!
//
NativeSize = GenCallJmpfDirect2(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)(ULONGLONG) EP,
0);
}
return NativeSize;
}
ULONG
PatchJmpfDirect(
PTHREADSTATE cpu,
IN PULONG pIntelDest,
IN PULONG PatchAddr
)
/*++
Routine Description:
This routine patches a JMPFDIRECT to a JMPFDIRECT2. It is called when
the native destination address of a jmp instruction is not yet known.
It patches the jmp to jump directly to the corresponding native code.
Arguments:
PatchAddr -- address of the JMPDIRECT in the Translation Cache
pIntelDest -- intel address of the destination of the jmp
Return Value:
Native address to jump to in order to resume execution
--*/
{
DWORD TCTimestamp;
ULONG NativeDest;
PVOID IntelDest;
USHORT Sel;
ULONG NativeSize;
if (cpu->flag_tf) {
return (ULONG)(ULONGLONG)&EndTranslatedCode;
}
//
// Switch from being a TC reader to a writer
//
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
//
// Compile the destination of the jmp and get the TC write lock
//
IntelDest = (PVOID)*(UNALIGNED DWORD *)pIntelDest;
Sel = *(UNALIGNED PUSHORT)(pIntelDest+1);
eip = (ULONG)(ULONGLONG) IntelDest;
CS = Sel;
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip(IntelDest, TRUE)->nativeStart;
if (TCTimestamp == TranslationCacheTimestamp) {
PULONG CodeLocation = PatchAddr - CallJmpfDirect_PATCHRA_OFFSET;
//
// The Translation Cache wasn't flushed - replace the JMPFDIRECT
// fragment by JMPFDIRECT2
//
NativeSize = GenCallJmpfDirect2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
(ULONG)NativeDest,
0);
NtFlushInstructionCache(
NtCurrentProcess(),
(PVOID)CodeLocation,
NativeSize
);
} else {
TCTimestamp = TranslationCacheTimestamp;
if (cpu->CSTimestamp != TCTimestamp) {
//
// The cache was flushed by another thread in the small window
// between mrsw calls in this thread, we plan on jumping directly
// to NativeDest, so the CPU callstack needs to be flushed.
// Normally this would be done in the CpuSimulate() loop as a
// result of jumping to EndTranslatedCode.
//
FlushCallstack(cpu);
}
}
//
// Switch back to being a TC reader
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
if (TCTimestamp == TranslationCacheTimestamp) {
//
// TC was not flushed - nativedest is valid
//
return NativeDest;
} else {
//
// TC was flushed while becomming a TC reader again - nativedest
// is now bogus.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
ULONG
PlaceCallDirect(
IN PULONG CodeLocation,
#if _ALPHA_
IN ULONG CurrentECU,
#endif
IN PINSTRUCTION Instruction
)
/*++
Routine Description:
This routine copies the unconditional call fragment into place.
Arguments:
Instruction - Supplies a description of the instruction the fragment
represents
CodeLocation - Supplies the address the code for the fragment has been
copied to
Return Value:
Size of code placed at CodeLocation
--*/
{
DWORD IntelDest;
DWORD IntelNext;
PENTRYPOINT EP;
// ASSUME: The first argument is always an immediate
CPUASSERT( Instruction->Operand1.Type == OPND_IMM );
IntelDest = Instruction->Operand1.Immed;
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
if (EP == NULL) {
//
// Knowing NativeDest requires compilation. Just place the unpatched
// version for now and patch it later if necessary
//
GenCallDirect(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
} else {
//
// We can place the patched version right away! Deterime if the
// NativeNext address is known.
//
IntelNext = Instruction->Operand2.Immed;
//
// If the current instruction is not the last one compiled, then
// NativeNext is CodeLocation+CallDirect_SIZE and CallDirect3 can
// be placed right away.
//
if (Instruction != &InstructionStream[NumberOfInstructions-1]) {
GenCallDirect3(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)(ULONGLONG) EP,
(ULONG)(ULONGLONG) (CodeLocation+CallDirect_SIZE));
} else {
GenCallDirect2(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)(ULONGLONG) EP,
0);
}
}
return CallDirect_SIZE * sizeof(ULONG);
}
ULONG
PlaceCallfDirect(
IN PULONG CodeLocation,
#if _ALPHA_
IN ULONG CurrentECU,
#endif
IN PINSTRUCTION Instruction
)
/*++
Routine Description:
This routine copies the unconditional FAR call fragment into place.
Arguments:
Instruction - Supplies a description of the instruction the fragment
represents
CodeLocation - Supplies the address the code for the fragment has been
copied to
Return Value:
Size of code placed at CodeLocation
--*/
{
DWORD IntelDest;
DWORD IntelAddr;
DWORD IntelNext;
PVOID NativeNext;
PENTRYPOINT EP;
// ASSUME: The first two arguments are pIntelDest and IntelNext, stored
// as immediates.
CPUASSERT( Instruction->Operand1.Type == OPND_IMM );
CPUASSERT( Instruction->Operand2.Type == OPND_IMM );
IntelAddr = Instruction->Operand1.Immed;
// Get the offset portion of the address (skipping the selector)
IntelDest = *(UNALIGNED DWORD *)(IntelAddr+2);
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
if (EP == NULL) {
//
// Knowing NativeDest requires compilation. Just place the unpatched
// version for now and patch it later if necessary
//
GenCallfDirect(CodeLocation,
#if _ALPHA_
CurrentECU,
#endif
Instruction);
} else {
//
// We can place the patched version right away! Deterime if the
// NativeNext address is known.
//
IntelNext = Instruction->Operand2.Immed;
//
// If the current instruction is not the last one compiled, then
// NativeNext is CodeLocation+CallfDirect_SIZE and CallDirect3 can
// be placed right away.
//
if (Instruction != &InstructionStream[NumberOfInstructions-1]) {
GenCallfDirect3(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)(ULONGLONG) EP,
(ULONG)(ULONGLONG) (CodeLocation+CallfDirect_SIZE));
} else {
GenCallfDirect2(CodeLocation,
TRUE, // compiling, not patching
#if _ALPHA_
CurrentECU,
#endif
(ULONG)(ULONGLONG) EP,
0);
}
}
return CallfDirect_SIZE * sizeof(ULONG);
}
DWORD
PatchCallDirectExceptionFilter(
PTHREADSTATE cpu
)
/*++
Routine Description:
Called if CTRL_CallFrag() throws an exception from within
PatchCallDirect(). If this happens, the Translation Cache is in fact
unlocked, although cpu->fTCUnlocked == FALSE. Need to fix this up before
CpuResetToConsistentState() gets run and unlocks the cache a second time.
Arguments:
cpu
Return Value:
None.
--*/
{
//
// Indicate the TC read lock is not held.
//
cpu->fTCUnlocked = TRUE;
//
// Continue unwinding the stack
//
return EXCEPTION_CONTINUE_SEARCH;
}
ULONG
PatchCallDirect(
IN PTHREADSTATE Cpu,
IN ULONG IntelDest,
IN ULONG IntelNext,
IN PULONG PatchAddr
)
/*++
Routine Description:
This routine patches a CALLDIRECT to a CALLDIRECT2. It is called when
the native destination address of a call instruction is not yet known.
It patches the call to jump directly to the corresponding native code.
Arguments:
Cpu -- per-thread info
IntelDest -- intel address of the destination of the call
IntelNext -- intel address of the instruction following the call
PatchAddr -- address of the CALLDIRECT in the Translation Cache
Return Value:
Native address to jump to in order to resume execution
--*/
{
DWORD TCTimestamp;
ULONG NativeDest;
ULONG NativeSize;
//
// Switch from being a TC reader to a writer
//
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
//
// Compile the code at the destination of the call and get the TC write lock
//
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
if (TCTimestamp == TranslationCacheTimestamp) {
PULONG CodeLocation = PatchAddr - CallDirect_PATCHRA_OFFSET;
//
// The Translation Cache wasn't flushed - replace the CALLDIRECT
// fragment by CALLDIRECT2
//
NativeSize = GenCallDirect2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
NativeDest,
0);
NtFlushInstructionCache(
NtCurrentProcess(),
CodeLocation,
NativeSize
);
} else {
TCTimestamp = TranslationCacheTimestamp;
if (Cpu->CSTimestamp != TCTimestamp) {
//
// The cache was flushed by another thread in the small window
// between mrsw calls in this thread, we plan on jumping directly
// to NativeDest, so the CPU callstack needs to be flushed.
// Normally this would be done in the CpuSimulate() loop as a
// result of jumping to EndTranslatedCode.
//
FlushCallstack(Cpu);
}
}
//
// Push IntelNext on the stack and update the stack optimization code.
// This must be done while still in TC write mode. If it isn't, then
// the transition back to TC reader may allow a TC cache flush, invalidating
// nativedest before it is written into the stack optimization. (The
// stack optimization is blown away whenever the TC is flushed, so if
// it is written in BEFORE the flush, it will just get blown away.
//
try {
CTRL_CallFrag(Cpu, IntelDest, IntelNext, 0 /* nativenext is unknown */);
} _except(PatchCallDirectExceptionFilter(Cpu)) {
// nothing to do - the exception filter does everything
}
//
// Become a TC reader again.
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
Cpu->fTCUnlocked = FALSE;
if (TCTimestamp == TranslationCacheTimestamp) {
//
// TC was not flushed while becomming a reader again.
//
return NativeDest;
} else {
//
// TC was flushed - nativedest is invalid. The callstack optimization
// was deleted when the TC flush occurred, so do an EndTranslatedCode
// instead.
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
ULONG
PatchCallfDirect(
IN PTHREADSTATE cpu,
IN PUSHORT pIntelDest,
IN ULONG IntelNext,
IN PULONG PatchAddr
)
/*++
Routine Description:
This routine patches a CALLFDIRECT to a CALLFDIRECT2. It is called when
the native destination address of a call instruction is not yet known.
It patches the call to jump directly to the corresponding native code.
Arguments:
cpu -- per-thread info
pIntelDest-- ptr to SEL:OFFSET intel address of the destination of the call
IntelNext -- intel address of the instruction following the call
PatchAddr -- address of the CALLDIRECT in the Translation Cache
Return Value:
Native address to jump to in order to resume execution
--*/
{
DWORD TCTimestamp;
ULONG NativeDest;
PVOID IntelDest;
ULONG NativeSize;
//
// Switch from being a TC reader to a writer
//
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
//
// Compile the code at the destination of the call and get the TC write lock
//
IntelDest = (PVOID)*(UNALIGNED DWORD *)pIntelDest;
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip(IntelDest, TRUE)->nativeStart;
if (TCTimestamp == TranslationCacheTimestamp) {
PULONG CodeLocation = PatchAddr - CallfDirect_PATCHRA_OFFSET;
//
// The Translation Cache wasn't flushed - replace the CALLDIRECT
// fragment by CALLDIRECT2
//
NativeSize = GenCallfDirect2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
NativeDest,
0);
NtFlushInstructionCache(
NtCurrentProcess(),
CodeLocation,
NativeSize
);
} else {
TCTimestamp = TranslationCacheTimestamp;
if (cpu->CSTimestamp != TCTimestamp) {
//
// The cache was flushed by another thread in the small window
// between mrsw calls in this thread, we plan on jumping directly
// to NativeDest, so the CPU callstack needs to be flushed.
// Normally this would be done in the CpuSimulate() loop as a
// result of jumping to EndTranslatedCode.
//
FlushCallstack(cpu);
}
}
//
// Push IntelNext on the stack and update the stack optimization code.
// This must be done while still in TC write mode. If it isn't, then
// the transition back to TC reader may allow a TC cache flush, invalidating
// nativedest before it is written into the stack optimization. (The
// stack optimization is blown away whenever the TC is flushed, so if
// it is written in BEFORE the flush, it will just get blown away.
//
try {
CTRL_CallfFrag(cpu, pIntelDest, IntelNext, 0 /* nativenext is unknown */);
} _except(PatchCallDirectExceptionFilter(cpu)) {
// nothing to do - the exception filter does everything
}
//
// Become a TC reader again.
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
cpu->fTCUnlocked = FALSE;
if (TCTimestamp == TranslationCacheTimestamp) {
//
// TC was not flushed while becomming a reader again.
//
return NativeDest;
} else {
//
// TC was flushed - nativedest is invalid. The callstack optimization
// was deleted when the TC flush occurred, so do an EndTranslatedCode
// instead.
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
ULONG
PatchCallDirect2(
IN PTHREADSTATE Cpu,
IN ULONG IntelDest,
IN ULONG IntelNext,
IN ULONG NativeDest,
IN PULONG PatchAddr
)
/*++
Routine Description:
This routine patches a CALLDIRECT2 to a CALLDIRECT3. It is called when
the native destination address of the instruction after the call is not yet
known. It patches the fragment to place the native address of the
instruction after the call on the optimized callstack.
Arguments:
Cpu -- per-thread info
IntelDest -- intel address of the destination of the call
IntelNext -- intel address of the instruction following the call
NativeDest -- native address of the destination of the call
PatchAddr -- address of the CALLDIRECT2 in the Translation Cache
Return Value:
Native address to jump to in order to resume execution
--*/
{
DWORD TCTimestamp;
ULONG NativeNext;
ULONG NativeSize;
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
NativeNext = (ULONG)(ULONGLONG) NativeAddressFromEipNoCompile((PVOID)IntelNext);
if (NativeNext) {
PULONG CodeLocation;
//
// The code at the return address from the call has already been
// compiled. Replace CALLDIRECT2 by CALLDIRECT3. TC is locked
// for write.
//
if (TCTimestamp != TranslationCacheTimestamp) {
//
// The TC was flushed while switching from reader to writer.
// Become a TC reader again
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
// The CALLDIRECT2 code is now gone, so set up for the call and
// then go to EndTranslatedCode to make the control transfer.
//
CTRL_CallFrag(Cpu,
IntelDest,
IntelNext,
0 // nativenext is also unknown
);
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
CodeLocation = PatchAddr - CallDirect2_PATCHRA_OFFSET;
//
// Else the TC was not flushed, and nativenext is now known. Patch
// CALLDIRECT2 to be CALLDIRECT3
//
NativeSize = GenCallDirect3(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
NativeDest,
NativeNext);
NtFlushInstructionCache(
NtCurrentProcess(),
CodeLocation,
NativeSize
);
//
// Push IntelNext on the stack and update the stack optimization code.
// This must be done while still in TC write mode. If it isn't, then
// the transition back to TC reader may allow a TC cache flush, invalidating
// nativedest before it is written into the stack optimization. (The
// stack optimization is blown away whenever the TC is flushed, so if
// it is written in BEFORE the flush, it will just get blown away.
//
CTRL_CallFrag(Cpu, IntelDest, IntelNext, NativeNext);
//
// Switch back to being a TC reader
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
if (TCTimestamp != TranslationCacheTimestamp) {
//
// TC was flushed while we were becomming a reader again.
// nativedest and nativenext are invalid, but stack optimization
// code was flushed.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
} else { // NativeNext == NULL, TC locked for Read
CTRL_CallFrag(Cpu, IntelDest, IntelNext, 0);
if (TCTimestamp != TranslationCacheTimestamp) {
//
// Cache was flushed by another thread. NativeDest is invalid.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
return NativeDest;
}
ULONG
PatchCallfDirect2(
IN PTHREADSTATE Cpu,
IN PUSHORT pIntelDest,
IN ULONG IntelNext,
IN ULONG NativeDest,
IN PULONG PatchAddr
)
/*++
Routine Description:
This routine patches a CALLFDIRECT2 to a CALLFDIRECT3. It is called when
the native destination address of the instruction after the call is not yet
known. It patches the fragment to place the native address of the
instruction after the call on the optimized callstack.
Arguments:
Cpu -- per-thread info
pIntelDest-- ptr to SEL:OFFSET intel address of the destination of the call
IntelNext -- intel address of the instruction following the call
NativeDest -- native address of the destination of the call
PatchAddr -- address of the CALLDIRECT2 in the Translation Cache
Return Value:
Native address to jump to in order to resume execution
--*/
{
DWORD TCTimestamp;
ULONG NativeNext;
ULONG NativeSize;
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
NativeNext = (ULONG)(ULONGLONG) NativeAddressFromEipNoCompile((PVOID)IntelNext);
if (NativeNext) {
PULONG CodeLocation;
//
// The code at the return address from the call has already been
// compiled. Replace CALLDIRECT2 by CALLDIRECT3. TC is locked
// for write.
//
if (TCTimestamp != TranslationCacheTimestamp) {
//
// The TC was flushed while switching from reader to writer.
// Become a TC reader again
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
// The CALLFDIRECT2 code is now gone, so set up for the call and
// then go to EndTranslatedCode to make the control transfer.
//
CTRL_CallfFrag(Cpu,
pIntelDest,
IntelNext,
0 // nativenext is also unknown
);
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
CodeLocation = PatchAddr - CallfDirect2_PATCHRA_OFFSET;
//
// Else the TC was not flushed, and nativenext is now known. Patch
// CALLFDIRECT2 to be CALLFDIRECT3
//
NativeSize = GenCallfDirect3(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
NativeDest,
NativeNext);
NtFlushInstructionCache(
NtCurrentProcess(),
CodeLocation,
NativeSize
);
//
// Push IntelNext on the stack and update the stack optimization code.
// This must be done while still in TC write mode. If it isn't, then
// the transition back to TC reader may allow a TC cache flush, invalidating
// nativedest before it is written into the stack optimization. (The
// stack optimization is blown away whenever the TC is flushed, so if
// it is written in BEFORE the flush, it will just get blown away.
//
CTRL_CallfFrag(Cpu, pIntelDest, IntelNext, NativeNext);
//
// Switch back to being a TC reader
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
if (TCTimestamp != TranslationCacheTimestamp) {
//
// TC was flushed while we were becomming a reader again.
// nativedest and nativenext are invalid, but stack optimization
// code was flushed.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
} else { // NativeNext == NULL, TC locked for Read
CTRL_CallfFrag(Cpu, pIntelDest, IntelNext, 0);
if (TCTimestamp != TranslationCacheTimestamp) {
//
// Cache was flushed by another thread. NativeDest is invalid.
//
return (ULONG)(ULONGLONG) &EndTranslatedCode;
}
}
return NativeDest;
}
DWORD
PatchCallIndirectExceptionFilter(
PTHREADSTATE cpu
)
/*++
Routine Description:
Called if CTRL_CallFrag() throws an exception from within
PatchCallIndirect(). If this happens, the Translation Cache Write lock
is being held. This must be released before the exception can be
allowed to continue.
Arguments:
cpu
Return Value:
None.
--*/
{
//
// Release the TC write lock.
//
MrswWriterExit(&MrswTC);
//
// Indicate the TC read lock is not held, either
//
cpu->fTCUnlocked = TRUE;
//
// Continue unwinding the stack
//
return EXCEPTION_CONTINUE_SEARCH;
}
VOID
PatchCallIndirect(
IN PTHREADSTATE Cpu,
IN ULONG IntelDest,
IN ULONG IntelNext,
IN PULONG PatchAddr
)
/*++
Routine Description:
This routine patches a CALLINDIRECT to a CALLINDIRECT2. It is called when
the native destination address of the instruction after the call is not yet
known. It patches the fragment to place the native address of the
instruction after the call on the optimized callstack.
Arguments:
Cpu -- per-thread info
IntelDest -- intel address of the destination of the call
IntelNext -- intel address of the instruction following the call
PatchAddr -- address of the CALLDIRECT2 in the Translation Cache
Return Value:
None. cpu->Eip updated to be IntelDest.
--*/
{
DWORD TCTimestamp;
ULONG NativeNext;
ULONG NativeSize;
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
NativeNext = (ULONG)(ULONGLONG) NativeAddressFromEipNoCompile((PVOID)IntelNext);
if (NativeNext) {
PULONG CodeLocation;
//
// The code at the return address from the call has already been
// compiled. Replace CALLINDIRECT by CALLINDIRECT2. TC is locked
// for write.
//
if (TCTimestamp != TranslationCacheTimestamp) {
//
// The TC was flushed while switching from reader to writer.
// Become a TC reader again
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
// The CALLINDIRECT code is now gone, so set up for the call
// without patching anything
//
CTRL_CallFrag(
Cpu,
IntelDest,
IntelNext,
0 // nativenext is unknown
);
return;
}
//
// Else the TC was not flushed, and nativenext is now known. Patch
// CALLINDIRECT to be CALLINDIRECT2
//
CodeLocation = PatchAddr - CallIndirect_PATCHRA_OFFSET;
NativeSize = GenCallIndirect2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
NativeNext,
getUniqueIndex());
NtFlushInstructionCache(
NtCurrentProcess(),
CodeLocation,
NativeSize
);
//
// Push IntelNext on the stack and update the stack optimization code.
// This must be done while still in TC write mode. If it isn't, then
// the transition back to TC reader may allow a TC cache flush, invalidating
// nativedest before it is written into the stack optimization. (The
// stack optimization is blown away whenever the TC is flushed, so if
// it is written in BEFORE the flush, it will just get blown away.
//
_try {
CTRL_CallFrag(Cpu, IntelDest, IntelNext, NativeNext);
} _except(PatchCallIndirectExceptionFilter(Cpu)) {
// nothing to do - the exception filter does everything
}
//
// Switch back to being a TC reader. TC flushes during the switch
// are OK and require no extra work.
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
} else { // NativeNext == NULL, TC locked for read.
CTRL_CallFrag(Cpu, IntelDest, IntelNext, 0);
}
return;
}
VOID
PatchCallfIndirect(
IN PTHREADSTATE Cpu,
IN PUSHORT pIntelDest,
IN ULONG IntelNext,
IN PULONG PatchAddr
)
/*++
Routine Description:
This routine patches a CALLFINDIRECT to a CALLFINDIRECT2. It is called when
the native destination address of the instruction after the call is not yet
known. It patches the fragment to place the native address of the
instruction after the call on the optimized callstack.
Arguments:
Cpu -- per-thread info
pIntelDest-- ptr to SEL:OFFSET intel address of the destination of the call
IntelNext -- intel address of the instruction following the call
PatchAddr -- address of the CALLDIRECT2 in the Translation Cache
Return Value:
None. cpu->eip updated to IntelDest
--*/
{
DWORD TCTimestamp;
ULONG NativeNext;
ULONG IntelDest;
ULONG NativeSize;
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
IntelDest = *(UNALIGNED DWORD *)pIntelDest;
NativeNext = (ULONG)(ULONGLONG) NativeAddressFromEipNoCompile((PVOID)IntelNext);
if (NativeNext) {
PULONG CodeLocation;
//
// The code at the return address from the call has already been
// compiled. Replace CALLINDIRECT by CALLINDIRECT2. TC is locked
// for write.
//
if (TCTimestamp != TranslationCacheTimestamp) {
//
// The TC was flushed while switching from reader to writer.
// Become a TC reader again
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
// The CALLFINDIRECT code is now gone, so set up for the call
// without patching anything
//
CTRL_CallfFrag(
Cpu,
pIntelDest,
IntelNext,
0 // nativenext is unknown
);
return;
}
//
// Else the TC was not flushed, and nativenext is now known. Patch
// CALLFINDIRECT to be CALLFINDIRECT2
//
CodeLocation = PatchAddr - CallfIndirect_PATCHRA_OFFSET;
NativeSize = GenCallfIndirect2(CodeLocation,
FALSE, // patching, not compiling
#if _ALPHA_
GetCurrentECU(CodeLocation),
#endif
NativeNext,
getUniqueIndex());
NtFlushInstructionCache(
NtCurrentProcess(),
CodeLocation,
NativeSize
);
//
// Push IntelNext on the stack and update the stack optimization code.
// This must be done while still in TC write mode. If it isn't, then
// the transition back to TC reader may allow a TC cache flush, invalidating
// nativedest before it is written into the stack optimization. (The
// stack optimization is blown away whenever the TC is flushed, so if
// it is written in BEFORE the flush, it will just get blown away.
//
_try {
CTRL_CallfFrag(Cpu, pIntelDest, IntelNext, NativeNext);
} _except(PatchCallIndirectExceptionFilter(Cpu)) {
// nothing to do - the exception filter does everything
}
//
// Switch back to being a TC reader. TC flushes during the switch
// are OK and require no extra work.
//
MrswWriterExit(&MrswTC);
MrswReaderEnter(&MrswTC);
} else { // NativeNext == NULL, TC locked for read.
CTRL_CallfFrag(Cpu, pIntelDest, IntelNext, 0);
}
return;
}
//*********************************************************************************
// Below are functions for the Indirect Control Transfer Table
//*********************************************************************************
// This number must be below 0xffff, because we want to be able to load it with just
// one instruction (now we use ori). It should also be a factor of two to get good
// code generation for % (so that we won't have to use a division instruction).
#define MAX_TABLE_ENTRIES 0x1000
typedef struct _IndirControlTransferTable {
ULONG intelAddr;
ULONG nativeAddr;
} INDIRCONTROLTRANSFERTABLE, *PINDIRCONTROLTRANSFERTABLE;
INDIRCONTROLTRANSFERTABLE IndirControlTransferTable[MAX_TABLE_ENTRIES];
// The last used index in the table
ULONG lastTableIndex;
ULONG
getUniqueIndex(
VOID
)
/*++
Routine Description:
This function returns the next free index to the indirect control
transfer table. If it reaches the end of the table, it wraps around.
NOTE: we need not worry about synchronization here, because we have
an Entry Point write lock whenever we are called.
Arguments:
none
Return Value:
An index into the table
--*/
{
return (lastTableIndex = ((lastTableIndex + 1) % MAX_TABLE_ENTRIES));
}
VOID
FlushIndirControlTransferTable(
VOID
)
/*++
Routine Description:
This routine flushes the Indirect Control Transfer Table
NOTE: we need not worry about synchronizations here, because the routine
which calls us (FlushTranslationCache) has a Translation Cache write lock.
Arguments:
none
Return Value:
none
--*/
{
RtlZeroMemory (IndirControlTransferTable, sizeof(INDIRCONTROLTRANSFERTABLE)*MAX_TABLE_ENTRIES);
lastTableIndex = 0;
}
ULONG
IndirectControlTransfer(
IN ULONG tableEntry,
IN ULONG intelAddr,
IN PTHREADSTATE cpu
)
/*++
Routine Description:
This routine is used by an indirect control transfer operation to try
and save a call to the Entry Point Manager.
Arguments:
tableEntry -- The index of the table entry where information might be
available about intelAddr
intelAddr -- The intel address we want to go to
cpu -- pointer to per-thread CPU data
Return Value:
The native address we want to go to
--*/
{
ULONG nativeAddr;
DWORD TCTimestamp;
//
// Detect apps which do the following:
// call foo
// where
// foo: mov eax, [esp]
// ...
// jmp eax ; this is really a 'ret' instruction
//
// This is the way _alloca() works - you call it with eax==number of bytes
// to allocate, and it jumps back to its caller with esp munged.
//
// What happens is the callstack cache gets out-of-sync. If the app
// is trying to do an indirect jump to the address already on the
// callstack cache, we will pop the callstack cache.
//
if (ISTOPOF_CALLSTACK(intelAddr)) {
POP_CALLSTACK(intelAddr, nativeAddr);
if (nativeAddr) {
return nativeAddr;
}
}
// First see if the table is filled in correctly already
MrswReaderEnter(&MrswIndirTable);
if (IndirControlTransferTable[tableEntry].intelAddr == intelAddr){
nativeAddr = IndirControlTransferTable[tableEntry].nativeAddr;
if (nativeAddr) {
MrswReaderExit(&MrswIndirTable);
return nativeAddr;
}
}
MrswReaderExit(&MrswIndirTable);
// Give up the translation cache reading lock so that we can call NativeAddressFromEip
TCTimestamp = TranslationCacheTimestamp;
MrswReaderExit(&MrswTC);
nativeAddr = (ULONG) (ULONGLONG)NativeAddressFromEip((PVOID)intelAddr, FALSE)->nativeStart;
// Note: we now have a TC read lock obtained by NativeAddressFromEip.
if (TCTimestamp == TranslationCacheTimestamp) {
// We haven't flushed the cache. Save the native address in the table.
MrswWriterEnter(&MrswIndirTable);
IndirControlTransferTable[tableEntry].intelAddr = intelAddr;
IndirControlTransferTable[tableEntry].nativeAddr = nativeAddr;
MrswWriterExit(&MrswIndirTable);
} else {
//
// Translation cache was flushed, possibly by another thread.
// Flush our callstack before resuming execution of RISC code
// in the Translation Cache.
//
FlushCallstack(cpu);
}
// Return the native address to IndirectControlTransferHelper which will go there.
return nativeAddr;
}
ULONG
IndirectControlTransferFar(
IN PTHREADSTATE cpu,
IN PUSHORT pintelAddr,
IN ULONG tableEntry
)
/*++
Routine Description:
This routine is used by a FAR indirect control transfer operation to try
and save a call to the Entry Point Manager.
Arguments:
tableEntry -- The index of the table entry where information might be
available about intelAddr
pintelAddr -- Pointer to SEL:OFFSET intel address we want to go to
Return Value:
The native address we want to go to
--*/
{
USHORT Sel;
ULONG Offset;
Offset = *(UNALIGNED PULONG)pintelAddr;
Sel = *(UNALIGNED PUSHORT)(pintelAddr+2);
CS = Sel;
eip = Offset;
return IndirectControlTransfer(tableEntry, Offset, cpu);
}
ULONG
PlaceNop(
IN PULONG CodeLocation,
#if _ALPHA_
IN ULONG CurrentECU,
#endif
IN PINSTRUCTION Instruction
)
{
return 0;
}
#if _ALPHA_
ULONG
GetCurrentECU(
PULONG CodeLocation
)
/*++
Routine Description:
This routine returns the correct ECU. CurrentECU is the target for
branch instructions when a fragment wants to jump to EndTranslatedCode.
N.B. This routine cannot change the global CurrentECU. This is set in
Compile(), and is the only way to locate the ECU at the end of
the translation cache if the exception info hasn't been placed
yet.
Arguments:
CodeLocation -- The code location which will be patched.
Return Value:
None.
--*/
{
//
// Find an EndCompilationUnit fragment by searching the Translation Cache
// for the next EXCEPTIONDATA_SIGNATURE. The code immediately before it
// is an EndCompilationUnit fragment.
//
while (*CodeLocation != EXCEPTIONDATA_SIGNATURE) {
CodeLocation++;
}
return (ULONG)(ULONGLONG)(CodeLocation-EndCompilationUnit_SIZE);
}
#endif // _ALPHA_-only
Reference in a new issue Copy permalink