354 lines
8.4 KiB
NASM
354 lines
8.4 KiB
NASM
|
title "Zero memory pages using fastest means available"
|
|||
|
|
;++
|
|||
|
|
;
|
|||
|
|
; Copyright (c) 1998 Microsoft Corporation
|
|||
|
|
;
|
|||
|
|
; Module Name:
|
|||
|
|
;
|
|||
|
|
; zero.asm
|
|||
|
|
;
|
|||
|
|
; Abstract:
|
|||
|
|
;
|
|||
|
|
; Zero memory pages using the fastest means available.
|
|||
|
|
;
|
|||
|
|
; Author:
|
|||
|
|
;
|
|||
|
|
; Peter Johnston (peterj) 20-Jun-1998.
|
|||
|
|
; Critical sections of Katmai code adapted from in-line
|
|||
|
|
; assembly version by Shiv Kaushik or Intel Corp.
|
|||
|
|
;
|
|||
|
|
; Environment:
|
|||
|
|
;
|
|||
|
|
; x86
|
|||
|
|
;
|
|||
|
|
; Revision History:
|
|||
|
|
;
|
|||
|
|
;--
|
|||
|
|
|
|||
|
|
.386p
|
|||
|
|
.xlist
|
|||
|
|
include ks386.inc
|
|||
|
|
include callconv.inc
|
|||
|
|
include mac386.inc
|
|||
|
|
.list
|
|||
|
|
|
|||
|
|
;
|
|||
|
|
; Register Definitions (for instruction macros).
|
|||
|
|
;
|
|||
|
|
|
|||
|
|
rEAX equ 0
|
|||
|
|
rECX equ 1
|
|||
|
|
rEDX equ 2
|
|||
|
|
rEBX equ 3
|
|||
|
|
rESP equ 4
|
|||
|
|
rEBP equ 5
|
|||
|
|
rESI equ 6
|
|||
|
|
rEDI equ 7
|
|||
|
|
|
|||
|
|
;
|
|||
|
|
; Define SIMD instructions used in this module.
|
|||
|
|
;
|
|||
|
|
|
|||
|
|
if 0
|
|||
|
|
|
|||
|
|
; these remain for reference only. In theory the stuff following
|
|||
|
|
; should generate the right code.
|
|||
|
|
|
|||
|
|
xorps_xmm0_xmm0 macro
|
|||
|
|
db 0FH, 057H, 0C0H
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
movntps_edx macro Offset
|
|||
|
|
db 0FH, 02BH, 042H, Offset
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
movaps_esp_xmm0 macro
|
|||
|
|
db 0FH, 029H, 004H, 024H
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
movaps_xmm0_esp macro
|
|||
|
|
db 0FH, 028H, 004H, 024H
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
endif
|
|||
|
|
|
|||
|
|
xorps macro XMMReg1, XMMReg2
|
|||
|
|
db 0FH, 057H, 0C0H + (XMMReg1 * 8) + XMMReg2
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
movntps macro GeneralReg, Offset, XMMReg
|
|||
|
|
db 0FH, 02BH, 040H + (XmmReg * 8) + GeneralReg, Offset
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
sfence macro
|
|||
|
|
db 0FH, 0AEH, 0F8H
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
movaps_load macro XMMReg, GeneralReg
|
|||
|
|
db 0FH, 028H, (XMMReg * 8) + 4, (4 * 8) + GeneralReg
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
movaps_store macro GeneralReg, XMMReg
|
|||
|
|
db 0FH, 029H, (XMMReg * 8) + 4, (4 * 8) + GeneralReg
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
|
|||
|
|
;
|
|||
|
|
; NPX Save and Restore
|
|||
|
|
;
|
|||
|
|
|
|||
|
|
fxsave macro Register
|
|||
|
|
db 0FH, 0AEH, Register
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
fxrstor macro Register
|
|||
|
|
db 0FH, 0AEH, 8+Register
|
|||
|
|
endm
|
|||
|
|
|
|||
|
|
|
|||
|
|
_TEXT SEGMENT DWORD PUBLIC 'CODE'
|
|||
|
|
ASSUME DS:FLAT, ES:FLAT, SS:NOTHING, FS:NOTHING, GS:NOTHING
|
|||
|
|
|
|||
|
|
;++
|
|||
|
|
;
|
|||
|
|
; VOID
|
|||
|
|
; KeZeroPage(
|
|||
|
|
; PageBase
|
|||
|
|
; )
|
|||
|
|
;
|
|||
|
|
; Routine Description:
|
|||
|
|
;
|
|||
|
|
; KeZeroPage is really just a function pointer that points at
|
|||
|
|
; either KiZeroPage or KiXMMIZeroPage depending on whether or
|
|||
|
|
; not XMMI instructions are available.
|
|||
|
|
;
|
|||
|
|
; Arguments:
|
|||
|
|
;
|
|||
|
|
; (ecx) PageBase Base address of page to be zeroed.
|
|||
|
|
;
|
|||
|
|
;
|
|||
|
|
; Return Value:
|
|||
|
|
;
|
|||
|
|
;--
|
|||
|
|
|
|||
|
|
|
|||
|
|
page ,132
|
|||
|
|
subttl "KiXMMIZeroPageNoSave - Use XMMI to zero memory (XMMI owned)"
|
|||
|
|
|
|||
|
|
;++
|
|||
|
|
;
|
|||
|
|
; VOID
|
|||
|
|
; KiXMMIZeroPageNoSave (
|
|||
|
|
; IN PVOID PageBase
|
|||
|
|
; )
|
|||
|
|
;
|
|||
|
|
; Routine Description:
|
|||
|
|
;
|
|||
|
|
; Use XMMI to zero a page of memory 16 bytes at a time while
|
|||
|
|
; at the same time minimizing cache polution.
|
|||
|
|
;
|
|||
|
|
; Note: The XMMI register set belongs to this thread. It is neither
|
|||
|
|
; saved nor restored by this procedure.
|
|||
|
|
;
|
|||
|
|
; Arguments:
|
|||
|
|
;
|
|||
|
|
; (ecx) PageBase Virtual address of the base of the page to be zeroed.
|
|||
|
|
;
|
|||
|
|
; Return Value:
|
|||
|
|
;
|
|||
|
|
; None.
|
|||
|
|
;
|
|||
|
|
;--
|
|||
|
|
|
|||
|
|
INNER_LOOP_BYTES equ 64
|
|||
|
|
|
|||
|
|
cPublicFastCall KiXMMIZeroPageNoSave,1
|
|||
|
|
cPublicFpo 0, 1
|
|||
|
|
|
|||
|
|
xorps 0, 0 ; zero xmm0 (128 bits)
|
|||
|
|
mov eax, PAGE_SIZE/INNER_LOOP_BYTES ; Number of Iterations
|
|||
|
|
|
|||
|
|
inner:
|
|||
|
|
|
|||
|
|
movntps rECX, 0, 0 ; store bytes 0 - 15
|
|||
|
|
movntps rECX, 16, 0 ; 16 - 31
|
|||
|
|
movntps rECX, 32, 0 ; 32 - 47
|
|||
|
|
movntps rECX, 48, 0 ; 48 - 63
|
|||
|
|
|
|||
|
|
add ecx, 64 ; increment base
|
|||
|
|
dec eax ; decrement loop count
|
|||
|
|
jnz short inner
|
|||
|
|
|
|||
|
|
; Force all stores to complete before any other
|
|||
|
|
; stores from this processor.
|
|||
|
|
|
|||
|
|
sfence
|
|||
|
|
|
|||
|
|
ifndef SFENCE_IS_NOT_BUSTED
|
|||
|
|
|
|||
|
|
; ERRATA the next uncached write to this processor's apic
|
|||
|
|
; may fail unless the store pipes have drained. sfence by
|
|||
|
|
; itself is not enough. Force drainage now by doing an
|
|||
|
|
; interlocked exchange.
|
|||
|
|
|
|||
|
|
xchg [esp-4], eax
|
|||
|
|
|
|||
|
|
endif
|
|||
|
|
|
|||
|
|
fstRET KiXMMIZeroPageNoSave
|
|||
|
|
|
|||
|
|
fstENDP KiXMMIZeroPageNoSave
|
|||
|
|
|
|||
|
|
|
|||
|
|
page ,132
|
|||
|
|
subttl "KiXMMIZeroPage - Use XMMI to zero memory"
|
|||
|
|
|
|||
|
|
;++
|
|||
|
|
;
|
|||
|
|
; VOID
|
|||
|
|
; KiXMMIZeroPage (
|
|||
|
|
; IN PVOID PageBase
|
|||
|
|
; )
|
|||
|
|
;
|
|||
|
|
; Routine Description:
|
|||
|
|
;
|
|||
|
|
; Use XMMI to zero a page of memory 16 bytes at a time. This
|
|||
|
|
; routine is a wrapper around KiXMMIZeroPageNoSave. In this
|
|||
|
|
; case we don't have the luxury of not saving/restoring context.
|
|||
|
|
;
|
|||
|
|
; Arguments:
|
|||
|
|
;
|
|||
|
|
; (ecx) PageBase Virtual address of the base of the page to be zeroed.
|
|||
|
|
;
|
|||
|
|
; Return Value:
|
|||
|
|
;
|
|||
|
|
; None.
|
|||
|
|
;
|
|||
|
|
;--
|
|||
|
|
|
|||
|
|
cPublicFastCall KiXMMIZeroPage,1
|
|||
|
|
cPublicFpo 0, 2
|
|||
|
|
|
|||
|
|
mov eax, PCR[PcInitialStack]
|
|||
|
|
mov edx, PCR[PcPrcbData+PbCurrentThread]
|
|||
|
|
push ebp
|
|||
|
|
push ebx
|
|||
|
|
mov ebp, esp ; save stack pointer
|
|||
|
|
sub esp, 16 ; reserve space for xmm0
|
|||
|
|
and esp, 0FFFFFFF0H ; 16 byte aligned
|
|||
|
|
cli ; don't context switch
|
|||
|
|
test [eax].FpCr0NpxState, CR0_EM ; if FP explicitly disabled
|
|||
|
|
jnz short kxzp90 ; do it the old way
|
|||
|
|
cmp byte ptr [edx].ThNpxState, NPX_STATE_LOADED
|
|||
|
|
je short kxzp80 ; jiff, NPX stated loaded
|
|||
|
|
|
|||
|
|
; NPX state is not loaded on this thread, it will be by
|
|||
|
|
; the time we reenable context switching.
|
|||
|
|
|
|||
|
|
mov byte ptr [edx].ThNpxState, NPX_STATE_LOADED
|
|||
|
|
|
|||
|
|
; enable use of FP instructions
|
|||
|
|
|
|||
|
|
mov ebx, cr0
|
|||
|
|
and ebx, NOT (CR0_MP+CR0_TS+CR0_EM)
|
|||
|
|
mov cr0, ebx ; enable NPX
|
|||
|
|
|
|||
|
|
ifdef NT_UP
|
|||
|
|
|
|||
|
|
; if this is a UP machine, the state might be loaded for
|
|||
|
|
; another thread in which case it needs to be saved.
|
|||
|
|
|
|||
|
|
mov ebx, PCR[PcPrcbData+PbNpxThread]; Owner of NPX state
|
|||
|
|
or ebx, ebx ; NULL?
|
|||
|
|
jz short @f ; yes, skip save.
|
|||
|
|
|
|||
|
|
mov byte ptr [ebx].ThNpxState, NPX_STATE_NOT_LOADED
|
|||
|
|
mov ebx, [ebx].ThInitialStack ; get address of save
|
|||
|
|
sub ebx, NPX_FRAME_LENGTH ; area.
|
|||
|
|
fxsave rEBX ; save NPX
|
|||
|
|
@@:
|
|||
|
|
|
|||
|
|
endif
|
|||
|
|
|
|||
|
|
; Now load the NPX context for this thread. This is because
|
|||
|
|
; if we switch away from this thread it will get saved again
|
|||
|
|
; in this save area and destroying it would be bad.
|
|||
|
|
|
|||
|
|
fxrstor rEAX
|
|||
|
|
|
|||
|
|
mov PCR[PcPrcbData+PbNpxThread], edx
|
|||
|
|
|
|||
|
|
kxzp80:
|
|||
|
|
sti ; reenable context switching
|
|||
|
|
movaps_store rESP, 0 ; save xmm0
|
|||
|
|
fstCall KiXMMIZeroPageNoSave ; zero the page
|
|||
|
|
movaps_load 0, rESP ; restore xmm
|
|||
|
|
|
|||
|
|
; restore stack pointer, non-volatiles and return
|
|||
|
|
|
|||
|
|
mov esp, ebp
|
|||
|
|
pop ebx
|
|||
|
|
pop ebp
|
|||
|
|
fstRET KiXMMIZeroPage
|
|||
|
|
|
|||
|
|
|
|||
|
|
; FP is explicitly disabled for this thread (probably a VDM
|
|||
|
|
; thread). Restore stack pointer, non-volatiles and jump into
|
|||
|
|
; KiZeroPage to do the work the old fashioned way.
|
|||
|
|
|
|||
|
|
kxzp90:
|
|||
|
|
sti
|
|||
|
|
mov esp, ebp
|
|||
|
|
pop ebx
|
|||
|
|
pop ebp
|
|||
|
|
jmp short @KiZeroPage@4
|
|||
|
|
|
|||
|
|
fstENDP KiXMMIZeroPage
|
|||
|
|
|
|||
|
|
|
|||
|
|
page ,132
|
|||
|
|
subttl "KiZeroPage - Available to all X86 processors"
|
|||
|
|
|
|||
|
|
;++
|
|||
|
|
;
|
|||
|
|
; KiZeroPage(
|
|||
|
|
; PVOID PageBase
|
|||
|
|
; )
|
|||
|
|
;
|
|||
|
|
; Routine Description:
|
|||
|
|
;
|
|||
|
|
; Generic Zero Page routine, used on processors that don't have
|
|||
|
|
; a more effecient way to zero large blocks of memory.
|
|||
|
|
; (Same as RtlZeroMemory).
|
|||
|
|
;
|
|||
|
|
; Arguments:
|
|||
|
|
;
|
|||
|
|
; (ecx) PageBase Base address of page to be zeroed.
|
|||
|
|
;
|
|||
|
|
; Return Value:
|
|||
|
|
;
|
|||
|
|
; None.
|
|||
|
|
;
|
|||
|
|
;--
|
|||
|
|
|
|||
|
|
cPublicFastCall KiZeroPage,1
|
|||
|
|
cPublicFpo 0, 0
|
|||
|
|
|
|||
|
|
push edi ; save EDI (non-volatile)
|
|||
|
|
xor eax, eax ; 32 bit zero
|
|||
|
|
mov edi, ecx ; setup for repsto
|
|||
|
|
mov ecx, PAGE_SIZE/4 ; iteration count
|
|||
|
|
|
|||
|
|
; store eax, ecx times starting at edi
|
|||
|
|
|
|||
|
|
rep stosd
|
|||
|
|
|
|||
|
|
pop edi ; restore edi and return
|
|||
|
|
fstRET KiZeroPage
|
|||
|
|
|
|||
|
|
fstENDP KiZeroPage
|
|||
|
|
|
|||
|
|
|
|||
|
|
_TEXT ends
|
|||
|
|
end
|