| Server IP : 121.121.20.254 / Your IP : 216.73.216.70 Web Server : Microsoft-IIS/10.0 System : Windows NT WEB-SERVER 10.0 build 20348 (Windows Server 2022) AMD64 User : IUSR ( 0) PHP Version : 8.3.28 Disable Function : NONE MySQL : ON | cURL : ON | WGET : OFF | Perl : OFF | Python : OFF | Sudo : OFF | Pkexec : OFF Directory : C:/Program Files (x86)/Windows Kits/10/Include/10.0.26100.0/shared/ |
Upload File : |
;++
;
; Copyright (c) Microsoft Corporation. All rights reserved.
;
;
; Module:
;
; kxarm64.w
;
; Abstract:
;
; Contains ARM architecture constants and assembly macros.
;
;--
;
; The ARM assembler uses a baroque syntax that is documented as part
; of the online Windows CE documentation. The syntax derives from
; ARM's own assembler and was chosen to allow the migration of
; specific assembly code bases, namely ARM's floating point runtime.
; While this compatibility is no longer strictly necessary, the
; syntax lives on....
;
; Highlights:
; * Assembler is white space sensitive. Symbols are defined by putting
; them in the first column
; * The macro definition mechanism is very primitive
;
; To augment the assembler, assembly files are run through CPP (as they are
; on IA64). This works well for constants but not structural components due
; to the white space sensitivity.
;
; For now, we use a mix of native assembler and CPP macros.
;
#undef TRUE
#undef FALSE
#define TRUE 1
#define FALSE 0
#include "kxarm64unw.h"
;
; For assembly files that are built for both ARM64 and ARM64EC (discouraged
; since the files might not have been ported to use X64 behavior in the ARM64EC
; paths), use this macro to wrap all references to function names (ASM and C).
;
; For ARM64, this does nothing.
;
; For ARM64EC, this changes FuncName to |#FuncName|.
;
#ifndef A64NAME
#define A64NAME_HASHLIT #
#define A64NAME_HASH() A64NAME_HASHLIT
#ifndef _M_ARM64EC
#define A64NAME(FuncName) FuncName
#else
#define A64NAME_DECORATE(FuncName) A64NAME_HASH() ## FuncName
#define A64NAME(FuncName) |A64NAME_DECORATE(FuncName)|
#endif /*_M_ARM64EC*/
#endif /*A64NAME*/
;
; Global variables
;
; Current function names and labels
GBLS __FuncNameNoBars
GBLS __FuncStartLabel
GBLS __FuncEpilog1StartLabel
GBLS __FuncEpilog2StartLabel
GBLS __FuncEpilog3StartLabel
GBLS __FuncEpilog4StartLabel
GBLS __FuncPDataLabel
GBLS __FuncXDataLabel
GBLS __FuncXDataPrologLabel
GBLS __FuncXDataEpilog1Label
GBLS __FuncXDataEpilog2Label
GBLS __FuncXDataEpilog3Label
GBLS __FuncXDataEpilog4Label
GBLS __FuncXDataEndLabel
GBLS __FuncEndLabel
GBLS __FuncEntryThunkLabel
GBLS __FuncExitThunkLabel
; other globals relating to the current function
GBLS __FuncComDat
GBLS __FuncArea
GBLS __FuncPDataArea
GBLS __FuncXDataArea
GBLA __FuncAlignment
__FuncAlignment SETA 4
;
; Helper macro: generate the various labels we will use internally
; for a function
;
; Output is placed in the various __Func*Label globals
;
MACRO
__DeriveFunctionLabels $FuncName, $AreaName
__FuncNameNoBars SETS "$FuncName"
IF ("$FuncName":LEFT:1 == "|") && ("$FuncName":RIGHT:1 == "|")
__FuncNameNoBars SETS ("$FuncName":LEFT:(:LEN:"$FuncName" - 1):RIGHT:(:LEN:"$FuncName" - 2))
ENDIF
__FuncStartLabel SETS "|$__FuncNameNoBars|"
__FuncEndLabel SETS "|$__FuncNameNoBars._end|"
__FuncEpilog1StartLabel SETS "|$__FuncNameNoBars._epilog1_start|"
__FuncEpilog2StartLabel SETS "|$__FuncNameNoBars._epilog2_start|"
__FuncEpilog3StartLabel SETS "|$__FuncNameNoBars._epilog3_start|"
__FuncEpilog4StartLabel SETS "|$__FuncNameNoBars._epilog4_start|"
__FuncPDataLabel SETS "|$__FuncNameNoBars._pdata|"
__FuncXDataLabel SETS "|$__FuncNameNoBars._xdata|"
__FuncXDataPrologLabel SETS "|$__FuncNameNoBars._xdata_prolog|"
__FuncXDataEpilog1Label SETS "|$__FuncNameNoBars._xdata_epilog1|"
__FuncXDataEpilog2Label SETS "|$__FuncNameNoBars._xdata_epilog2|"
__FuncXDataEpilog3Label SETS "|$__FuncNameNoBars._xdata_epilog3|"
__FuncXDataEpilog4Label SETS "|$__FuncNameNoBars._xdata_epilog4|"
__FuncXDataEndLabel SETS "|$__FuncNameNoBars._xdata_end|"
__FuncEntryThunkLabel SETS "|$__FuncNameNoBars._entry_thunk|"
__FuncArea SETS "|.text|"
__FuncPDataArea SETS "|.pdata|"
__FuncXDataArea SETS "|.xdata|"
IF "$AreaName" != ""
__FuncArea SETS "$AreaName"
ENDIF
IF __FuncComDat != ""
__FuncArea SETS __FuncArea:CC:"{|$__FuncNameNoBars|}"
__FuncPDataArea SETS __FuncPDataArea:CC:"{$__FuncPDataLabel}"
__FuncXDataArea SETS __FuncXDataArea:CC:"{$__FuncXDataLabel}"
ENDIF
MEND
;
; Helper macro: create a global label for the given name,
; decorate it, and export it for external consumption.
;
MACRO
__ExportName $FuncName
LCLS Name
IF ("$FuncName":LEFT:1 == "|") && ("$FuncName":RIGHT:1 == "|")
Name SETS "$FuncName"
ELSE
Name SETS "|$FuncName|"
ENDIF
ALIGN 4
EXPORT $Name
$Name
MEND
MACRO
__ExportProc $FuncName
LCLS Name
Name SETS "|$FuncName|"
ALIGN 4
EXPORT $Name
$Name PROC
MEND
;
; Helper macro to set the AREA to the correct answer
; for the current function, and configure the alignment.
;
MACRO
__SetFunctionAreaAndAlign $Alignment
LCLS AreaAlign
LCLS AlignStmt
;
; "NOALIGN" is supported to just set the area
;
IF "$Alignment" == "NOALIGN"
AREA $__FuncArea
;
; COMDAT functions must set alignment in the AREA
; statement
;
ELIF __FuncComDat != ""
AreaAlign SETS "4"
IF "$Alignment" != ""
IF $Alignment > 4
AreaAlign SETS "$Alignment"
ENDIF
ENDIF
AREA $__FuncArea,CODE,READONLY,ALIGN=$AreaAlign
ELSE
AlignStmt SETS ""
IF "$Alignment" != ""
AlignStmt SETS "ALIGN 0x":CC: :STR:(1 << $Alignment)
ENDIF
AREA $__FuncArea,CODE,READONLY
$AlignStmt
ENDIF
MEND
;
; Helper macro to emit the special DWORD prior to the start of a
; function which contains an offset to the ARM64EC entry thunk.
; The entry thunk must have been previously defined or else this
; macro is a no-op.
;
MACRO
__AddEntryThunkPointer $Alignment
IF :DEF:$__FuncEntryThunkLabel
ASSERT(__FuncComDat == "")
ALIGN (1 << $Alignment),(1 << $Alignment) - 4
dcd ($__FuncEntryThunkLabel - ({PC} + 4)) + 1
ENDIF
MEND
;
; Declare that all following code/data is to be put in the .text segment
;
; N.B. The ALIGN attribute here specifies an exponent of base 2; not a
; direct byte count. Thus ALIGN=4 specifies a 16 byte alignment.
;
MACRO
TEXTAREA
AREA |.text|,ALIGN=4,CODE,READONLY
MEND
;
; Declare that all following code/data is to be put in the .data segment
;
MACRO
DATAAREA
AREA |.data|,DATA
MEND
;
; Declare that all following code/data is to be put in the .rdata segment
;
MACRO
RODATAAREA
AREA |.rdata|,DATA,READONLY
MEND
;
; Set/reset the alignment for COMDAT functions that follow.
;
MACRO
SET_COMDAT_ALIGNMENT $Alignment
__FuncAlignment SETA $Alignment
IF __FuncAlignment < 4
__FuncAlignment SETA 4
ENDIF
MEND
MACRO
RESET_COMDAT_ALIGNMENT
__FuncAlignment SETA 4
MEND
;
; Macro for indicating the start of a nested function. Nested functions
; imply a prolog, epilog, and unwind codes. This macro presumes that
; __DeriveFunctionLabels and __SetFunctionAreaAndAlign have already been
; called as appropriate
;
MACRO
NESTED_ENTRY $FuncName, $AreaName, $ExceptHandler, $Alignment
__FuncComDat SETS ""
__DeriveFunctionLabels $FuncName,$AreaName
__SetFunctionAreaAndAlign $Alignment
IF ("$Alignment" != "")
__FuncAlignment SETA $Alignment
ELSE
__FuncAlignment SETA 4
ENDIF
__AddEntryThunkPointer $__FuncAlignment
__ResetUnwindState $ExceptHandler
__ExportProc $__FuncNameNoBars
ROUT
MEND
MACRO
NESTED_ENTRY_COMDAT $FuncName, $AreaName, $ExceptHandler, $Alignment
IF ("$Alignment" != "")
SET_COMDAT_ALIGNMENT $Alignment
ENDIF
__FuncComDat SETS "COMDAT"
__DeriveFunctionLabels $FuncName,$AreaName
__SetFunctionAreaAndAlign $__FuncAlignment
ASSERT (!:DEF:$__FuncEntryThunkLabel)
__ResetUnwindState $ExceptHandler
__ExportProc $__FuncNameNoBars
ROUT
MEND
;
; Generate an ARM64EC entry thunk for an upcoming function.
; Note that only COMDAT functions are supported.
;
#if defined(_M_ARM64EC)
IMPORT __os_arm64x_dispatch_ret
MACRO
ARM64EC_ENTRY_THUNK $FuncName, $Parameters, $SaveQCount, $AreaName, $Alignment
LCLS OriginalFunc
LCLS OriginalArea
LCLA Params
LCLA QCount
Params SETA $Parameters
IF "$SaveQCount" == ""
QCount SETA 10
ELSE
QCount SETA (($SaveQCount + 1)/2)*2
ENDIF
EXPORT $FuncName
; first derive labels for the original function
; and set that as the target area
__FuncComDat SETS ""
__DeriveFunctionLabels $FuncName,$AreaName
__SetFunctionAreaAndAlign $Alignment
OriginalFunc SETS __FuncStartLabel
OriginalArea SETS __FuncArea
; then derive labels for the thunk
__DeriveFunctionLabels $__FuncEntryThunkLabel,$AreaName
__FuncArea SETS OriginalArea
__ResetUnwindState
__ExportProc $__FuncNameNoBars
PROLOG_SAVE_REG_PAIR fp, lr, #-16!
; load parameters into x4-x8 as appropriate
ASSERT(Params <= 8)
IF (Params >= 6)
PROLOG_NOP ldp x5, x6, [sp, #48]
ELIF (Params >= 5)
PROLOG_NOP ldr x5, [sp, #48]
ENDIF
IF (Params >= 8)
PROLOG_NOP ldp x7, x8, [sp, #64]
ELIF (Params >= 7)
PROLOG_NOP ldr x7, [sp, #64]
ENDIF
; optionally save XMMs
IF (QCount >= 10)
PROLOG_SAVE_REG_PAIR q6, q7, #-160!
PROLOG_SAVE_REG_PAIR q8, q9, #32
PROLOG_SAVE_REG_PAIR q10, q11, #64
PROLOG_SAVE_REG_PAIR q12, q13, #96
PROLOG_SAVE_REG_PAIR q14, q15, #128
ELIF (QCount >= 8)
PROLOG_SAVE_REG_PAIR q6, q7, #-128!
PROLOG_SAVE_REG_PAIR q8, q9, #32
PROLOG_SAVE_REG_PAIR q10, q11, #64
PROLOG_SAVE_REG_PAIR q12, q13, #96
ELIF (QCount >= 6)
PROLOG_SAVE_REG_PAIR q6, q7, #-96!
PROLOG_SAVE_REG_PAIR q8, q9, #32
PROLOG_SAVE_REG_PAIR q10, q11, #64
ELIF (QCount >= 4)
PROLOG_SAVE_REG_PAIR q6, q7, #-64!
PROLOG_SAVE_REG_PAIR q8, q9, #32
ELIF (QCount >= 2)
PROLOG_SAVE_REG_PAIR q6, q7, #-32!
ENDIF
; call through and copy return result to x8
bl $OriginalFunc
mov x8,x0
; optionally restore XMMs
IF (QCount >= 10)
EPILOG_RESTORE_REG_PAIR q14, q15, #128
EPILOG_RESTORE_REG_PAIR q12, q13, #96
EPILOG_RESTORE_REG_PAIR q10, q11, #64
EPILOG_RESTORE_REG_PAIR q8, q9, #32
EPILOG_RESTORE_REG_PAIR q6, q7, #160!
ELIF (QCount >= 8)
EPILOG_RESTORE_REG_PAIR q12, q13, #96
EPILOG_RESTORE_REG_PAIR q10, q11, #64
EPILOG_RESTORE_REG_PAIR q8, q9, #32
EPILOG_RESTORE_REG_PAIR q6, q7, #128!
ELIF (QCount >= 6)
EPILOG_RESTORE_REG_PAIR q10, q11, #64
EPILOG_RESTORE_REG_PAIR q8, q9, #32
EPILOG_RESTORE_REG_PAIR q6, q7, #96!
ELIF (QCount >= 4)
EPILOG_RESTORE_REG_PAIR q8, q9, #32
EPILOG_RESTORE_REG_PAIR q6, q7, #64!
ELIF (QCount >= 2)
EPILOG_RESTORE_REG_PAIR q6, q7, #32!
ENDIF
EPILOG_RESTORE_REG_PAIR fp, lr, #16!
EPILOG_NOP adrp x16, __os_arm64x_dispatch_ret
EPILOG_NOP ldr x16, [x16, __os_arm64x_dispatch_ret]
EPILOG_END br x16
NESTED_END
MEND
MACRO
ARM64EC_CUSTOM_ENTRY_THUNK $FuncName, $AreaName, $Alignment
LCLS OriginalFunc
LCLS OriginalArea
EXPORT $FuncName
; first derive labels for the original function
; and set that as the target area
__FuncComDat SETS ""
__DeriveFunctionLabels $FuncName,$AreaName
__SetFunctionAreaAndAlign $Alignment
OriginalFunc SETS __FuncStartLabel
OriginalArea SETS __FuncArea
; then derive labels for the thunk
__DeriveFunctionLabels $__FuncEntryThunkLabel,$AreaName
__FuncArea SETS OriginalArea
__ResetUnwindState
__ExportProc $__FuncNameNoBars
MEND
MACRO
ARM64EC_CUSTOM_ENTRY_THUNK_END
EPILOG_NOP adrp x16, __os_arm64x_dispatch_ret
EPILOG_NOP ldr x16, [x16, __os_arm64x_dispatch_ret]
EPILOG_END br x16
NESTED_END
MEND
#else
MACRO
ARM64EC_ENTRY_THUNK $FuncName, $Parameters, $SaveQCount, $AreaName, $Alignment
MEND
MACRO
ARM64EC_CUSTOM_ENTRY_THUNK $FuncName, $AreaName, $Alignment
MEND
#endif
;
; Macro for indicating the end of a nested function. We generate the
; .pdata and .xdata records here as necessary.
;
; Note that the $FuncName parameter is vestigial and not consumed.
;
MACRO
NESTED_END $FuncName
; mark the end of the function
$__FuncEndLabel
LTORG
ENDP
; generate .pdata
IF __FuncComDat != ""
AREA $__FuncPDataArea,ALIGN=2,READONLY,ASSOC=$__FuncArea
ELSE
AREA $__FuncPDataArea,ALIGN=2,READONLY
ENDIF
DCD $__FuncStartLabel
RELOC 2 ; make this relative to image base
DCD $__FuncXDataLabel
RELOC 2 ; make this relative to image base
; generate .xdata
__EmitUnwindXData
; back to the original area
__SetFunctionAreaAndAlign NOALIGN
; reset the labels
__FuncStartLabel SETS ""
__FuncEndLabel SETS ""
MEND
;
; Macro for indicating the start of a leaf function.
;
MACRO
LEAF_ENTRY $FuncName, $AreaName, $Alignment
NESTED_ENTRY $FuncName, $AreaName, "", $Alignment
MEND
MACRO
LEAF_ENTRY_COMDAT $FuncName, $AreaName, $Alignment
NESTED_ENTRY_COMDAT $FuncName, $AreaName, "", $Alignment
MEND
;
; Macro for indicating the end of a leaf function.
;
MACRO
LEAF_END $FuncName
NESTED_END $FuncName
MEND
#if defined(_CHPE_X86_ARM64_) || defined(ARM64_ASM_IN_COMDAT) || (defined(_M_ARM64EC) && defined (ARM64EC_ASM_IN_COMDAT))
#define NESTED_ENTRY NESTED_ENTRY_COMDAT
#define LEAF_ENTRY LEAF_ENTRY_COMDAT
#endif
#if !defined(_CHPE_X86_ARM64_)
;
; Macro for indicating the start of a leaf function.
;
MACRO
LEAF_ENTRY_NO_PDATA $FuncName, $AreaName
; compute the function's labels
__DeriveFunctionLabels $FuncName,$AreaName
__SetFunctionAreaAndAlign
; export the function name
__ExportProc $__FuncNameNoBars
; flush any pending literal pool stuff
ROUT
MEND
;
; Macro for indicating the end of a leaf function.
;
MACRO
LEAF_END_NO_PDATA $FuncName
; mark the end of the function
$__FuncEndLabel
LTORG
ENDP
; reset the labels
__FuncStartLabel SETS ""
__FuncEndLabel SETS ""
MEND
#endif
;
; Macro for indicating an alternate entry point into a function.
;
MACRO
ALTERNATE_ENTRY $FuncName
; export the entry point's name
__ExportName $FuncName
; flush any pending literal pool stuff
ROUT
MEND
;
; Macro for getting the address of a data item.
;
MACRO
ADDROF $Reg, $Variable
adrp $Reg, $Variable ; get the page address first
add $Reg, $Reg, $Variable ; add in the low bits
MEND
;
; Macro for loading a 32-bit constant.
;
MACRO
MOVL32 $Reg, $Variable
IF ((($Variable):SHR:16):AND:0xffff) == 0
movz $Reg, #$Variable
ELIF ((($Variable):SHR:0):AND:0xffff) == 0
movz $Reg, #((($Variable):SHR:16):AND:0xffff), lsl #16
ELSE
movz $Reg, #(($Variable):AND:0xffff)
movk $Reg, #((($Variable):SHR:16):AND:0xffff), lsl #16
ENDIF
MEND
#if defined(_CAPKERN)
;
; Icecap entrypoints
;
IMPORT _CAP_Start_Profiling
IMPORT _CAP_End_Profiling
;
; Macro to record a call record
;
MACRO
CAPSTART $arg1, $arg2
stp x0, x1, [sp, #-16]!
ldr x0, =$arg1
ldr x1, =$arg2
bl _CAP_Start_Profiling
ldp x0, x1, [sp], #16
MEND
;
; Macro to record a return record
;
MACRO
CAPEND $arg1
sub sp, sp, #16
str x0, [sp, #0]
ldr x0, =$arg1
bl _CAP_End_Profiling
ldr x0, [sp, #0]
add sp, sp, #16
MEND
#else
MACRO
CAPSTART $arg1, $arg2
MEND
MACRO
CAPEND $arg1
MEND
#endif
;
; Macro to align a Control Flow Guard valid call target.
; Not necessary to use this before functions anymore as
; it is the default for NESTED_ENTRY/LEAF_ENTRY macros.
;
MACRO
CFG_ALIGN
ALIGN 16
MEND
;
; Macro to perform a Control Flow Guard check on a live call target.
;
; $TargetReg - Target address register
; x16 - Bitmap address
; $FailLabel - Label to jump to in the event of a failure
;
; N.B. x16-x17 are free, other registers should be treated as live.
;
; N.B. ValidTarget should only specify a label for the ARM64EC checkers.
; The function has logic below which depends on this.
;
MACRO
CFG_ICALL_CHECK_BITMAP $TargetReg, $ValidTarget, $FailLabel
LCLS ValidLabel
ValidLabel SETS ""
IF ("$ValidTarget":LEFT:1 != "x" && "$ValidTarget":LEFT:1 != "w" && "$ValidTarget" != "lr")
ValidLabel SETS "$ValidTarget"
ENDIF
;
; Bitmap is an array of 2-bit values. Each 2-bit value represents 16 bytes, with the low
; bit set if jumps are permitted, and the upper bit set if misaligned jumps are permitted.
;
; The bit index is (address >> 4).
; Each byte holds 4 entries, so the byte index is (address >> 6).
; The shift amount is computed as 2*((address >> 4) & 3), or (address >> 3) & 6
; If address is aligned, (address >> 3) & 7 is equivalent, and ubfx can be used to extract.
;
lsr x17, $TargetReg, #6 ; compute bitmap byte index
tst $TargetReg, #15 ; misaligned address?
ldrb w17, [x16, x17] ; load byte from bitmap
ubfx x16, $TargetReg, #3, #3 ; compute bit index*2
bne %F2 ; if misaligned, account for extra bits
lsr x17, x17, x16 ; shift bitmap chunk over to valid align bit
tbz x17, #0, %F3 ; if low bit not set, verify the upper bit to
; allow an export suppressed target
1 ; Valid
;
; "ret lr" and "br lr" have different encodings and use of "ret lr" is
; preferred to hint that it is returning to the caller.
;
IF "$ValidLabel" != ""
b $ValidLabel ; jump to valid target
ELIF "$ValidTarget" != "lr"
br $ValidTarget ; jump to valid target
ELSE
ret lr ; return
ENDIF
2 ; Misaligned
;
; Code on ARM64 should always be 16 byte aligned if address taken.
;
; TODO: Compress bitmap format to 1 bit per address on ARM64?
; TODO: We are seeing 8-byte aligned code now. (1/22/14)
;
and x16, x16, #0xfffffffffffffffe ; force low bit of shift to 0
lsr x17, x17, x16 ; shift bitmap chunk down
tbz x17, #0, %F4 ; invalid if the low bit was clear
3 ; FailOpen
tbnz x17, #1, %B1 ; valid if upper bit was set as well
4 ; Failure
IF "$ValidLabel" != ""
;
; Don't need to explicitly set mode for ARM64EC.
;
ELIF "$ValidTarget" != "lr"
mov x16, #1 ; CFG dispatch mode
IF ("$TargetReg" != "x15")
mov x15, $TargetReg ; move CFG target to x15 for failure function
ENDIF
ELSE
mov x16, #0 ; CFG check mode
IF ("$TargetReg" != "x15")
mov x15, $TargetReg ; move CFG target to x15 for failure function
ENDIF
ENDIF
b $FailLabel ; jump to failure function
MEND
;
; Macro to perform a Control Flow Guard check on a live call target with export suppression.
;
; $TargetReg - Target address register
; x16 - Bitmap address
; $FailLabel - Label to jump to in the event of a failure
;
; N.B. x16-x17 are free, other registers should be treated as live.
;
; N.B. ValidTarget should only specify a label for the ARM64EC checkers.
; The function has logic below which depends on this.
;
MACRO
CFG_ICALL_CHECK_BITMAP_ES $TargetReg, $ValidTarget, $FailLabel
LCLS ValidLabel
ValidLabel SETS ""
IF ("$ValidTarget":LEFT:1 != "x" && "$ValidTarget":LEFT:1 != "w" && "$ValidTarget" != "lr")
ValidLabel SETS "$ValidTarget"
ENDIF
;
; Bitmap is an array of 2-bit values. Each 2-bit value represents 16 bytes, with the low
; bit set if jumps are permitted, and the upper bit set if misaligned jumps are permitted.
;
; The bit index is (address >> 4).
; Each byte holds 4 entries, so the byte index is (address >> 6).
; The shift amount is computed as 2*((address >> 4) & 3), or (address >> 3) & 6
; If address is aligned, (address >> 3) & 7 is equivalent, and ubfx can be used to extract.
;
lsr x17, $TargetReg, #6 ; compute bitmap byte index
tst $TargetReg, #15 ; misaligned address?
ldrb w17, [x16, x17] ; load byte from bitmap
ubfx x16, $TargetReg, #3, #3 ; compute bit index*2
bne %F2 ; if misaligned, account for extra bits
lsr x17, x17, x16 ; shift bitmap chunk over to valid align bit
tbz x17, #0, %F3 ; if low bit not set, either invalid or export
; suppressed target
1 ; Valid
;
; "ret lr" and "br lr" have different encodings and use of "ret lr" is
; preferred to hint that it is returning to the caller.
;
IF "$ValidLabel" != ""
b $ValidLabel ; jump to valid target
ELIF "$ValidTarget" != "lr"
br $ValidTarget ; jump to valid target
ELSE
ret lr ; return
ENDIF
2 ; Misaligned
;
; Code on ARM64 should always be 16 byte aligned if address taken.
;
; TODO: Compress bitmap format to 1 bit per address on ARM64?
; TODO: We are seeing 8-byte aligned code now. (1/22/14)
;
and x16, x16, #0xfffffffffffffffe ; force low bit of shift to 0
lsr x17, x17, x16 ; shift bitmap chunk down
tbz x17, #0, %F3 ; invalid if the low bit was clear
tbnz x17, #1, %B1 ; valid if upper bit was set as well
3 ; Failure
IF "$ValidLabel" != ""
;
; Don't need to explicitly set mode for ARM64EC.
;
ELIF "$ValidTarget" != "lr"
mov x16, #1 ; CFG dispatch mode
IF ("$TargetReg" != "x15")
mov x15, $TargetReg ; move CFG target to x15 for failure function
ENDIF
ELSE
mov x16, #0 ; CFG check mode
IF ("$TargetReg" != "x15")
mov x15, $TargetReg ; move CFG target to x15 for failure function
ENDIF
ENDIF
b $FailLabel ; jump to failure function
MEND
;
; Macro to acquire a spin lock at address $Reg + $Offset. Clobbers {r0-r2}
;
; ARM64_WORKITEM : should we use acquire/release semantics instead of DMB?
;
; TODO: Today this routine is not used. If it is used in the future, consider
; whether the yield should be switched to enlightened yield.
;
MACRO
ACQUIRE_SPIN_LOCK $Reg, $Offset
mov x0, #1 ; we want to exchange with a 1
dmb ; memory barrier ahead of the loop
1
ldxr x1, [$Reg, $Offset] ; load the new value
stxr x2, x0, [$Reg, $Offset] ; attempt to store the 1
cbnz x2, %B1 ; did we succeed before someone else did?
cbz x1, %F3 ; was the lock previously owned? if not, we're done
yield ; yield execution
b %B1 ; and try again
3
dmb
MEND
;
; Macro to release a spin lock at address $Reg + $Offset.
;
; ARM64_WORKITEM : should we use acquire/release semantics instead of DMB?
MACRO
RELEASE_SPIN_LOCK $Reg, $Offset
dmb
str xzr, [$Reg, $Offset] ; store 0
MEND
;
; Macro to increment a 64-bit statistic.
;
MACRO
INCREMENT_STAT $AddrReg, $Temp1, $Temp2
1 ldxr $Temp1, [$AddrReg] ; load current value
add $Temp1, $Temp1, #1 ; increment
stxr $Temp2, $Temp1, [$AddrReg] ; attempt to store
cbnz $Temp2, %B1 ; loop until it works?
MEND
;
; Macros to enable/disable interrupts.
;
MACRO
ENABLE_INTERRUPTS
msr DAIFClr, #2 ; enable interrupts
MEND
MACRO
DISABLE_INTERRUPTS
msr DAIFSet, #2 ; disable interrupts
MEND
;
; Macros to read/write the current IRQL
;
; N.B. These macros do not do hardware and software IRQL processing.
;
MACRO
GET_IRQL $Irql
ldrb $Irql, [x18, #PcCurrentIrql] ; read IRQL
MEND
MACRO
RAISE_IRQL $Reg, $NewIrql
#if DBG
GET_IRQL $Reg ; get old IRQL
cmp $Reg, #$NewIrql ; check if old IRQL is lower or same
bls %F1 ; if LS, valid transition
EMIT_BREAKPOINT ; break
1
#endif
mov $Reg, #$NewIrql ; get new IRQL
strb $Reg, [x18, #PcCurrentIrql] ; update IRQL
MEND
;
; Macros to output special undefined opcodes that indicate breakpoints
; and debug services.
;
MACRO
EMIT_BREAKPOINT
brk #ARM64_BREAKPOINT
MEND
MACRO
EMIT_DEBUG_SERVICE
brk #ARM64_DEBUG_SERVICE
MEND
MACRO
FASTFAIL $FastFailCode
mov x0, $FastFailCode
brk #ARM64_FASTFAIL
MEND
;
; Macro to generate an exception frame; this is intended to
; be used within the prolog of a function.
;
MACRO
GENERATE_EXCEPTION_FRAME
PROLOG_SAVE_REG_PAIR x19, x20, #-96!
PROLOG_SAVE_REG_PAIR x21, x22, #16
#if !defined(_M_ARM64EC)
PROLOG_SAVE_REG_PAIR x23, x24, #32
#else
PROLOG_NOP stp xzr, xzr, [sp, #32]
#endif
PROLOG_SAVE_REG_PAIR x25, x26, #48
#if !defined(_M_ARM64EC)
PROLOG_SAVE_REG_PAIR x27, x28, #64
#else
PROLOG_SAVE_REG x27, #64
PROLOG_NOP str xzr, [sp, #72]
#endif
PROLOG_SAVE_REG_PAIR fp, lr, #80
MEND
;
; Macro to restore from an exception frame; this is intended to
; be used within the epilog of a function.
;
MACRO
RESTORE_EXCEPTION_STATE
EPILOG_RESTORE_REG_PAIR fp, lr, #80
#if !defined(_M_ARM64EC)
EPILOG_RESTORE_REG_PAIR x27, x28, #64
#else
EPILOG_RESTORE_REG x27 , #64
#endif
EPILOG_RESTORE_REG_PAIR x25, x26, #48
#if !defined(_M_ARM64EC)
EPILOG_RESTORE_REG_PAIR x23, x24, #32
#endif
EPILOG_RESTORE_REG_PAIR x21, x22, #16
EPILOG_RESTORE_REG_PAIR x19, x20, #96!
MEND
;
; Macro to ensure that any eret is followed by barriers to
; prevent speculation
;
MACRO
ERET_FIX
eret
dsb sy
isb sy
DCD 0xD50330FF ; sb
MEND
#define eret ERET_FIX
;
; Given an address, obtains a pointer to the EC code bitmap from the
; specified global (typically located in the .mrdata section) and
; performs a bitmap lookup to determine if the address is EC code.
;
; Sets the Zero Flag for X64 targets, clears the Zero Flag for EC targets.
;
; xAddress - On input, the code address to be tested.
; This value is preserved.
;
; T1 - On input, the address of the EC Bitmap.
; This register is then used as scratch.
;
; T2 - On input, the max user-land address.
; This register is then used as scratch.
;
; xResult - Returns true(1) if the target is EC code
; and false(0) otherwise. This can be 'xzr'
; if a boolen result is not required. It can
; also overlap with xAddress, T1 or T2.
;
; SkipBoundsChecking - If set to "SkipBoundsChecking", no EC
; Bitmap bounds checks are performed (and
; T2 doesn't need to provide the max user-land
; address). T2 is still a scratch reg.
;
; Zero Flag - Z=0 for EC code and Z=1 otherwise.
;
MACRO
EC_BITMAP_LOOKUP $xAddress, $T1, $T2, $xResult, $SkipBoundsChecking
IF "$SkipBoundsChecking" != "SkipBoundsChecking"
cmp $xAddress, x$T2 ; Check if the address is above user space range
bhi %F1
cmp $xAddress, #(MM_LOWEST_USER_ADDRESS / 4096), lsl #12 ; Check if address < MM_LOWEST_USER_ADDRESS (64KiB)
blo %F1 ; if so, take the fast path
ENDIF
lsr x$T2, $xAddress, #15 ; each byte of bitmap indexes 8*4K = 2^15 byte span
ldrb w$T2, [x$T1, x$T2] ; load the bitmap byte for the 8*4K span
;
; * IF THIS INSTRUCTION EVER CHANGES, SO MUST
; KiOpPreprocessAccessViolation *
;
ubfx x$T1, $xAddress, #12, #3 ; index to the 4K page within the 8*4K span
lsr x$T1, x$T2, x$T1
IF "$SkipBoundsChecking" != "SkipBoundsChecking"
b %F2
1
mov x$T1, xzr
ENDIF
2
ands $xResult, x$T1, #1 ; test the specific page
MEND