;++
;
; Copyright (c) Microsoft Corporation.  All rights reserved.
;
;
; Module:
;
;   kxarm64.w
;
; Abstract:
;
;   Contains ARM architecture constants and assembly macros.
;
;--

;
; The ARM assembler uses a baroque syntax that is documented as part
; of the online Windows CE documentation.  The syntax derives from
; ARM's own assembler and was chosen to allow the migration of
; specific assembly code bases, namely ARM's floating point runtime.
; While this compatibility is no longer strictly necessary, the
; syntax lives on....
;
; Highlights:
;      * Assembler is white space sensitive.  Symbols are defined by putting
;        them in the first column
;      * The macro definition mechanism is very primitive
;
; To augment the assembler, assembly files are run through CPP (as they are
; on IA64).  This works well for constants but not structural components due
; to the white space sensitivity.
;
; For now, we use a mix of native assembler and CPP macros.
;

#undef TRUE
#undef FALSE

#define TRUE 1
#define FALSE 0

#include "kxarm64unw.h"


;
; For assembly files that are built for both ARM64 and ARM64EC (discouraged
; since the files might not have been ported to use X64 behavior in the ARM64EC
; paths), use this macro to wrap all references to function names (ASM and C).
;
; For ARM64, this does nothing.
;
; For ARM64EC, this changes FuncName to |#FuncName|.
;

#ifndef A64NAME
#define A64NAME_HASHLIT #
#define A64NAME_HASH() A64NAME_HASHLIT
#ifndef _M_ARM64EC
#define A64NAME(FuncName) FuncName
#else
#define A64NAME_DECORATE(FuncName) A64NAME_HASH() ## FuncName
#define A64NAME(FuncName) |A64NAME_DECORATE(FuncName)|  
#endif /*_M_ARM64EC*/
#endif /*A64NAME*/

        ;
        ; Global variables
        ;

        ; Current function names and labels
        GBLS    __FuncNameNoBars
        GBLS    __FuncStartLabel
        GBLS    __FuncEpilog1StartLabel
        GBLS    __FuncEpilog2StartLabel
        GBLS    __FuncEpilog3StartLabel
        GBLS    __FuncEpilog4StartLabel
        GBLS    __FuncPDataLabel
        GBLS    __FuncXDataLabel
        GBLS    __FuncXDataPrologLabel
        GBLS    __FuncXDataEpilog1Label
        GBLS    __FuncXDataEpilog2Label
        GBLS    __FuncXDataEpilog3Label
        GBLS    __FuncXDataEpilog4Label
        GBLS    __FuncXDataEndLabel
        GBLS    __FuncEndLabel
        GBLS    __FuncEntryThunkLabel
        GBLS    __FuncExitThunkLabel

        ; other globals relating to the current function
        GBLS    __FuncComDat
        GBLS    __FuncArea
        GBLS    __FuncPDataArea
        GBLS    __FuncXDataArea
        GBLA    __FuncAlignment
__FuncAlignment SETA 4

        ;
        ; Helper macro: generate the various labels we will use internally
        ; for a function
        ;
        ; Output is placed in the various __Func*Label globals
        ;

        MACRO
        __DeriveFunctionLabels $FuncName, $AreaName

__FuncNameNoBars        SETS "$FuncName"
        IF ("$FuncName":LEFT:1 == "|") && ("$FuncName":RIGHT:1 == "|")
__FuncNameNoBars        SETS ("$FuncName":LEFT:(:LEN:"$FuncName" - 1):RIGHT:(:LEN:"$FuncName" - 2))
        ENDIF
__FuncStartLabel        SETS "|$__FuncNameNoBars|"
__FuncEndLabel          SETS "|$__FuncNameNoBars._end|"
__FuncEpilog1StartLabel SETS "|$__FuncNameNoBars._epilog1_start|"
__FuncEpilog2StartLabel SETS "|$__FuncNameNoBars._epilog2_start|"
__FuncEpilog3StartLabel SETS "|$__FuncNameNoBars._epilog3_start|"
__FuncEpilog4StartLabel SETS "|$__FuncNameNoBars._epilog4_start|"
__FuncPDataLabel        SETS "|$__FuncNameNoBars._pdata|"
__FuncXDataLabel        SETS "|$__FuncNameNoBars._xdata|"
__FuncXDataPrologLabel  SETS "|$__FuncNameNoBars._xdata_prolog|"
__FuncXDataEpilog1Label SETS "|$__FuncNameNoBars._xdata_epilog1|"
__FuncXDataEpilog2Label SETS "|$__FuncNameNoBars._xdata_epilog2|"
__FuncXDataEpilog3Label SETS "|$__FuncNameNoBars._xdata_epilog3|"
__FuncXDataEpilog4Label SETS "|$__FuncNameNoBars._xdata_epilog4|"
__FuncXDataEndLabel     SETS "|$__FuncNameNoBars._xdata_end|"
__FuncEntryThunkLabel   SETS "|$__FuncNameNoBars._entry_thunk|"
__FuncArea              SETS "|.text|"
__FuncPDataArea         SETS "|.pdata|"
__FuncXDataArea         SETS "|.xdata|"
        IF "$AreaName" != ""
__FuncArea              SETS "$AreaName"
        ENDIF
        IF __FuncComDat != ""
__FuncArea              SETS __FuncArea:CC:"{|$__FuncNameNoBars|}"
__FuncPDataArea         SETS __FuncPDataArea:CC:"{$__FuncPDataLabel}"
__FuncXDataArea         SETS __FuncXDataArea:CC:"{$__FuncXDataLabel}"
        ENDIF

        MEND


        ;
        ; Helper macro: create a global label for the given name,
        ; decorate it, and export it for external consumption.
        ;

        MACRO
        __ExportName $FuncName

        LCLS    Name

        IF ("$FuncName":LEFT:1 == "|") && ("$FuncName":RIGHT:1 == "|")
Name    SETS    "$FuncName"
        ELSE
Name    SETS    "|$FuncName|"
        ENDIF

        ALIGN   4
        EXPORT  $Name
$Name
        MEND

        MACRO
        __ExportProc $FuncName

        LCLS    Name
Name    SETS    "|$FuncName|"
        ALIGN   4
        EXPORT  $Name
$Name   PROC
        MEND


        ;
        ; Helper macro to set the AREA to the correct answer
        ; for the current function, and configure the alignment.
        ;

        MACRO
        __SetFunctionAreaAndAlign $Alignment

        LCLS    AreaAlign
        LCLS    AlignStmt

        ;
        ; "NOALIGN" is supported to just set the area
        ;

        IF "$Alignment" == "NOALIGN"
        AREA    $__FuncArea

        ;
        ; COMDAT functions must set alignment in the AREA
        ; statement
        ;

        ELIF __FuncComDat != ""

AreaAlign SETS "4"
        IF "$Alignment" != ""
        IF $Alignment > 4
AreaAlign SETS "$Alignment"
        ENDIF
        ENDIF

        AREA    $__FuncArea,CODE,READONLY,ALIGN=$AreaAlign

        ELSE

AlignStmt SETS ""
        IF "$Alignment" != ""
AlignStmt SETS "ALIGN 0x":CC: :STR:(1 << $Alignment)
        ENDIF

        AREA    $__FuncArea,CODE,READONLY
        $AlignStmt

        ENDIF

        MEND


        ;
        ; Helper macro to emit the special DWORD prior to the start of a
        ; function which contains an offset to the ARM64EC entry thunk.
        ; The entry thunk must have been previously defined or else this
        ; macro is a no-op.
        ;

        MACRO
        __AddEntryThunkPointer

        IF :DEF:$__FuncEntryThunkLabel
        ASSERT(__FuncComDat != "")
        ALIGN   (1 << $__FuncAlignment),(1 << $__FuncAlignment) - 4
        dcd     ($__FuncEntryThunkLabel - ({PC} + 4)) + 1
        ENDIF

        MEND


        ;
        ; Declare that all following code/data is to be put in the .text segment
        ;
        ; N.B. The ALIGN attribute here specifies an exponent of base 2; not a
        ;      direct byte count. Thus ALIGN=4 specifies a 16 byte alignment.
        ;

        MACRO
        TEXTAREA
        AREA    |.text|,ALIGN=4,CODE,READONLY
        MEND


        ;
        ; Declare that all following code/data is to be put in the .data segment
        ;

        MACRO
        DATAAREA
        AREA    |.data|,DATA
        MEND


        ;
        ; Declare that all following code/data is to be put in the .rdata segment
        ;

        MACRO
        RODATAAREA
        AREA    |.rdata|,DATA,READONLY
        MEND


        ;
        ; Set/reset the alignment for COMDAT functions that follow.
        ;
        MACRO
        SET_COMDAT_ALIGNMENT $Alignment

__FuncAlignment SETA $Alignment
        IF __FuncAlignment < 4
__FuncAlignment SETA 4
        ENDIF

        MEND


        MACRO
        RESET_COMDAT_ALIGNMENT

__FuncAlignment SETA 4

        MEND


        ;
        ; Macro for indicating the start of a nested function. Nested functions
        ; imply a prolog, epilog, and unwind codes. This macro presumes that
        ; __DeriveFunctionLabels and __SetFunctionAreaAndAlign have already been
        ; called as appropriate
        ;

        MACRO
        NESTED_ENTRY $FuncName, $AreaName, $ExceptHandler, $Alignment

__FuncComDat SETS ""
        __DeriveFunctionLabels $FuncName,$AreaName
        __SetFunctionAreaAndAlign $Alignment
        ASSERT (!:DEF:$__FuncEntryThunkLabel)
        __ResetUnwindState $ExceptHandler
        __ExportProc $__FuncNameNoBars
        ROUT

        MEND


        MACRO
        NESTED_ENTRY_COMDAT $FuncName, $AreaName, $ExceptHandler, $Alignment

        IF ("$Alignment" != "")
        SET_COMDAT_ALIGNMENT $Alignment
        ENDIF

__FuncComDat SETS "COMDAT"
        __DeriveFunctionLabels $FuncName,$AreaName
        __SetFunctionAreaAndAlign $__FuncAlignment
        __AddEntryThunkPointer
        __ResetUnwindState $ExceptHandler
        __ExportProc $__FuncNameNoBars
        ROUT

        MEND


        ;
        ; Generate an ARM64EC entry thunk for an upcoming function.
        ; Note that only COMDAT functions are supported.
        ;

#if defined(_M_ARM64EC)
        IMPORT __os_arm64x_dispatch_ret

        MACRO
        ARM64EC_ENTRY_THUNK $FuncName, $Parameters, $SaveQCount, $AreaName

        LCLS    OriginalFunc
        LCLS    OriginalArea
        LCLA    Params
        LCLA    QCount

Params  SETA    $Parameters
        IF "$SaveQCount" == ""
QCount  SETA    10
        ELSE
QCount  SETA    (($SaveQCount + 1)/2)*2
        ENDIF

        EXPORT $FuncName

        ; first derive labels for the original function
        ; and set that as the target area
__FuncComDat SETS "COMDAT"
        __DeriveFunctionLabels $FuncName,$AreaName
        __SetFunctionAreaAndAlign $__FuncAlignment
OriginalFunc SETS __FuncStartLabel
OriginalArea SETS __FuncArea

        ; then derive labels for the thunk
        __DeriveFunctionLabels $__FuncEntryThunkLabel,$AreaName
__FuncArea SETS OriginalArea
        __ResetUnwindState
        __ExportProc $__FuncNameNoBars

        PROLOG_SAVE_REG_PAIR fp, lr, #-16!

        ; load parameters into x4-x8 as appropriate
        ASSERT(Params <= 8)
        IF (Params >= 6)
        PROLOG_NOP ldp x5, x6, [sp, #48]
        ELIF (Params >= 5)
        PROLOG_NOP ldr x5, [sp, #48]
        ENDIF
        IF (Params >= 8)
        PROLOG_NOP ldp x7, x8, [sp, #64]
        ELIF (Params >= 7)
        PROLOG_NOP ldr x7, [sp, #64]
        ENDIF

        ; optionally save XMMs
        IF (QCount >= 10)
        PROLOG_SAVE_REG_PAIR q6, q7, #-160!
        PROLOG_SAVE_REG_PAIR q8, q9, #32
        PROLOG_SAVE_REG_PAIR q10, q11, #64
        PROLOG_SAVE_REG_PAIR q12, q13, #96
        PROLOG_SAVE_REG_PAIR q14, q15, #128
        ELIF (QCount >= 8)
        PROLOG_SAVE_REG_PAIR q6, q7, #-128!
        PROLOG_SAVE_REG_PAIR q8, q9, #32
        PROLOG_SAVE_REG_PAIR q10, q11, #64
        PROLOG_SAVE_REG_PAIR q12, q13, #96
        ELIF (QCount >= 6)
        PROLOG_SAVE_REG_PAIR q6, q7, #-96!
        PROLOG_SAVE_REG_PAIR q8, q9, #32
        PROLOG_SAVE_REG_PAIR q10, q11, #64
        ELIF (QCount >= 4)
        PROLOG_SAVE_REG_PAIR q6, q7, #-64!
        PROLOG_SAVE_REG_PAIR q8, q9, #32
        ELIF (QCount >= 2)
        PROLOG_SAVE_REG_PAIR q6, q7, #-32!
        ENDIF

        ; call through and copy return result to x8
        bl      $OriginalFunc
        mov     x8,x0

        ; optionally restore XMMs
        IF (QCount >= 10)
        EPILOG_RESTORE_REG_PAIR q14, q15, #128
        EPILOG_RESTORE_REG_PAIR q12, q13, #96
        EPILOG_RESTORE_REG_PAIR q10, q11, #64
        EPILOG_RESTORE_REG_PAIR q8, q9, #32
        EPILOG_RESTORE_REG_PAIR q6, q7, #160!
        ELIF (QCount >= 8)
        EPILOG_RESTORE_REG_PAIR q12, q13, #96
        EPILOG_RESTORE_REG_PAIR q10, q11, #64
        EPILOG_RESTORE_REG_PAIR q8, q9, #32
        EPILOG_RESTORE_REG_PAIR q6, q7, #128!
        ELIF (QCount >= 6)
        EPILOG_RESTORE_REG_PAIR q10, q11, #64
        EPILOG_RESTORE_REG_PAIR q8, q9, #32
        EPILOG_RESTORE_REG_PAIR q6, q7, #96!
        ELIF (QCount >= 4)
        EPILOG_RESTORE_REG_PAIR q8, q9, #32
        EPILOG_RESTORE_REG_PAIR q6, q7, #64!
        ELIF (QCount >= 2)
        EPILOG_RESTORE_REG_PAIR q6, q7, #32!
        ENDIF

        EPILOG_RESTORE_REG_PAIR fp, lr, #16!

        EPILOG_NOP adrp    x16, __os_arm64x_dispatch_ret
        EPILOG_NOP ldr     x16, [x16, __os_arm64x_dispatch_ret]
        EPILOG_NOP br      x16

        NESTED_END

        MEND

        MACRO
        ARM64EC_CUSTOM_ENTRY_THUNK $FuncName, $AreaName

        LCLS    OriginalFunc
        LCLS    OriginalArea

        EXPORT $FuncName

        ; first derive labels for the original function
        ; and set that as the target area
__FuncComDat SETS "COMDAT"
        __DeriveFunctionLabels $FuncName,$AreaName
        __SetFunctionAreaAndAlign $__FuncAlignment
OriginalFunc SETS __FuncStartLabel
OriginalArea SETS __FuncArea

        ; then derive labels for the thunk
        __DeriveFunctionLabels $__FuncEntryThunkLabel,$AreaName
__FuncArea SETS OriginalArea
        __ResetUnwindState
        __ExportProc $__FuncNameNoBars

        MEND

        MACRO
        ARM64EC_CUSTOM_ENTRY_THUNK_END

        EPILOG_NOP adrp    x16, __os_arm64x_dispatch_ret
        EPILOG_NOP ldr     x16, [x16, __os_arm64x_dispatch_ret]
        EPILOG_NOP br      x16

        NESTED_END

        MEND

#else
        MACRO
        ARM64EC_ENTRY_THUNK $FuncName, $Parameters, $SaveQCount=10, $AreaName
        MEND
#endif


        ;
        ; Macro for indicating the end of a nested function. We generate the
        ; .pdata and .xdata records here as necessary.
        ;
        ; Note that the $FuncName parameter is vestigial and not consumed.
        ;

        MACRO
        NESTED_END $FuncName

        ; mark the end of the function
$__FuncEndLabel
        LTORG
        ENDP

        ; generate .pdata

        IF __FuncComDat != ""
        AREA    $__FuncPDataArea,ALIGN=2,READONLY,ASSOC=$__FuncArea
        ELSE
        AREA    $__FuncPDataArea,ALIGN=2,READONLY
        ENDIF

        DCD     $__FuncStartLabel
        RELOC   2                                       ; make this relative to image base

        DCD     $__FuncXDataLabel
        RELOC   2                                       ; make this relative to image base

        ; generate .xdata
        __EmitUnwindXData

        ; back to the original area
        __SetFunctionAreaAndAlign NOALIGN

        ; reset the labels
__FuncStartLabel SETS    ""
__FuncEndLabel  SETS    ""

        MEND


        ;
        ; Macro for indicating the start of a leaf function.
        ;

        MACRO
        LEAF_ENTRY $FuncName, $AreaName, $Alignment

        NESTED_ENTRY $FuncName, $AreaName, "", $Alignment

        MEND


        MACRO
        LEAF_ENTRY_COMDAT $FuncName, $AreaName, $Alignment

        NESTED_ENTRY_COMDAT $FuncName, $AreaName, "", $Alignment

        MEND


        ;
        ; Macro for indicating the end of a leaf function.
        ;

        MACRO
        LEAF_END $FuncName

        NESTED_END $FuncName

        MEND


#if defined(_CHPE_X86_ARM64_) || defined(ARM64_ASM_IN_COMDAT) || (defined(_M_ARM64EC) && defined (ARM64EC_ASM_IN_COMDAT))
#define NESTED_ENTRY NESTED_ENTRY_COMDAT
#define LEAF_ENTRY LEAF_ENTRY_COMDAT
#endif


#if !defined(_CHPE_X86_ARM64_)

        ;
        ; Macro for indicating the start of a leaf function.
        ;

        MACRO
        LEAF_ENTRY_NO_PDATA $FuncName, $AreaName

        ; compute the function's labels
        __DeriveFunctionLabels $FuncName,$AreaName
        __SetFunctionAreaAndAlign

        ; export the function name
        __ExportProc $__FuncNameNoBars

        ; flush any pending literal pool stuff
        ROUT

        MEND


        ;
        ; Macro for indicating the end of a leaf function.
        ;

        MACRO
        LEAF_END_NO_PDATA $FuncName

        ; mark the end of the function
$__FuncEndLabel
        LTORG
        ENDP

        ; reset the labels
__FuncStartLabel SETS    ""
__FuncEndLabel  SETS    ""

        MEND

#endif


        ;
        ; Macro for indicating an alternate entry point into a function.
        ;

        MACRO
        ALTERNATE_ENTRY $FuncName

        ; export the entry point's name
        __ExportName $FuncName

        ; flush any pending literal pool stuff
        ROUT

        MEND


        ;
        ; Macro for getting the address of a data item.
        ;
        
        MACRO
        ADDROF $Reg, $Variable
        
        adrp    $Reg, $Variable                 ; get the page address first
        add     $Reg, $Reg, $Variable           ; add in the low bits
        
        MEND


        ;
        ; Macro for loading a 32-bit constant.
        ;
        
        MACRO
        MOVL32 $Reg, $Variable
        
        IF ((($Variable):SHR:16):AND:0xffff) == 0
        movz    $Reg, #$Variable
        ELIF ((($Variable):SHR:0):AND:0xffff) == 0
        movz    $Reg, #((($Variable):SHR:16):AND:0xffff), lsl #16
        ELSE
        movz    $Reg, #(($Variable):AND:0xffff)
        movk    $Reg, #((($Variable):SHR:16):AND:0xffff), lsl #16
        ENDIF
        
        MEND


#if defined(_CAPKERN)

        ;
        ; Icecap entrypoints
        ;

        IMPORT _CAP_Start_Profiling
        IMPORT _CAP_End_Profiling


        ;
        ; Macro to record a call record
        ;

        MACRO
        CAPSTART $arg1, $arg2

        stp x0, x1, [sp, #-16]!

        ldr x0, =$arg1
        ldr x1, =$arg2
        bl _CAP_Start_Profiling

        ldp x0, x1, [sp], #16

        MEND


        ;
        ; Macro to record a return record
        ;

        MACRO
        CAPEND $arg1

        sub sp, sp, #16
        str x0, [sp, #0]

        ldr x0, =$arg1
        bl _CAP_End_Profiling

        ldr x0, [sp, #0]
        add sp, sp, #16

        MEND

#else

        MACRO
        CAPSTART $arg1, $arg2
        MEND

        MACRO
        CAPEND $arg1
        MEND

#endif


        ;
        ; Macro to align a Control Flow Guard valid call target.
        ; Not necessary to use this before functions anymore as
        ; it is the default for NESTED_ENTRY/LEAF_ENTRY macros.
        ;

        MACRO
        CFG_ALIGN
        ALIGN 16
        MEND


        ;
        ; Macro to perform a Control Flow Guard check on a live call target.
        ;
        ; $TargetReg - Target address register
        ; x16 - Bitmap address
        ; $FailLabel - Label to jump to in the event of a failure
        ; 
        ; N.B. x16-x17 are free, other registers should be treated as live.
        ;
        ; N.B. This macro expects the $FailLabel to be right after this macro.
        ;

        MACRO
        CFG_ICALL_CHECK_BITMAP $TargetReg, $ValidTarget, $FailLabel

        LCLS    ValidLabel

ValidLabel SETS ""

        IF ("$ValidTarget":LEFT:1 != "x" && "$ValidTarget":LEFT:1 != "w" && "$ValidTarget" != "lr")

ValidLabel SETS "$ValidTarget"

        ENDIF

;
; Bitmap is an array of 2-bit values. Each 2-bit value represents 16 bytes, with the low
; bit set if jumps are permitted, and the upper bit set if misaligned jumps are permitted.
;
; The bit index is (address >> 4).
; Each byte holds 4 entries, so the byte index is (address >> 6).
; The shift amount is computed as 2*((address >> 4) & 3), or (address >> 3) & 6
; If address is aligned, (address >> 3) & 7 is equivalent, and ubfx can be used to extract.
;

        lsr     x17, $TargetReg, #6             ; compute bitmap byte index
        tst     $TargetReg, #15                 ; misaligned address?
        ldrb    w17, [x16, x17]                 ; load byte from bitmap
        ubfx    x16, $TargetReg, #3, #3         ; compute bit index*2
        bne     %F2                             ; if misaligned, account for extra bits
        lsr     x17, x17, x16                   ; shift bitmap chunk over to valid align bit
        tbz     x17, #0, %F3                    ; if low bit not set, verify the upper bit to
                                                ; allow an export suppressed target
1 ; Valid

;
; "ret lr" and "br lr" have different encodings and use of "ret lr" is
; preferred to hint that it is returning to the caller.
;

        IF "$ValidLabel" != ""
            b       $ValidLabel                 ; jump to valid target
        ELIF "$ValidTarget" != "lr"
            br      $ValidTarget                ; jump to valid target
        ELSE
            ret     lr                          ; return
        ENDIF

2 ; Misaligned
        ;
        ; Code on ARM64 should always be 16 byte aligned if address taken.
        ;
        ; TODO:  Compress bitmap format to 1 bit per address on ARM64?
        ; TODO:  We are seeing 8-byte aligned code now. (1/22/14)
        ;
        and     x16, x16, #0xfffffffffffffffe   ; force low bit of shift to 0
        lsr     x17, x17, x16                   ; shift bitmap chunk down
        tbz     x17, #0, $FailLabel             ; invalid if the low bit was clear
3 ; FailOpen
        tbnz    x17, #1, %B1                    ; valid if upper bit was set as well

        MEND


        ;
        ; Macro to perform a Control Flow Guard check on a live call target with export suppression.
        ;
        ; $TargetReg - Target address register
        ; x16 - Bitmap address
        ; $FailLabel - Label to jump to in the event of a failure
        ; 
        ; N.B. x16-x17 are free, other registers should be treated as live.
        ;
        ; N.B. This macro expects the $FailLabel to be right after this macro.
        ;

        MACRO
        CFG_ICALL_CHECK_BITMAP_ES $TargetReg, $ValidTarget, $FailLabel

        LCLS    ValidLabel

ValidLabel SETS ""

        IF ("$ValidTarget":LEFT:1 != "x" && "$ValidTarget":LEFT:1 != "w" && "$ValidTarget" != "lr")

ValidLabel SETS "$ValidTarget"

        ENDIF

;
; Bitmap is an array of 2-bit values. Each 2-bit value represents 16 bytes, with the low
; bit set if jumps are permitted, and the upper bit set if misaligned jumps are permitted.
;
; The bit index is (address >> 4).
; Each byte holds 4 entries, so the byte index is (address >> 6).
; The shift amount is computed as 2*((address >> 4) & 3), or (address >> 3) & 6
; If address is aligned, (address >> 3) & 7 is equivalent, and ubfx can be used to extract.
;

        lsr     x17, $TargetReg, #6             ; compute bitmap byte index
        tst     $TargetReg, #15                 ; misaligned address?
        ldrb    w17, [x16, x17]                 ; load byte from bitmap
        ubfx    x16, $TargetReg, #3, #3         ; compute bit index*2
        bne     %F2                             ; if misaligned, account for extra bits
        lsr     x17, x17, x16                   ; shift bitmap chunk over to valid align bit
        tbz     x17, #0, $FailLabel             ; if low bit not set, either invalid or export
                                                ; suppressed target
1 ; Valid

;
; "ret lr" and "br lr" have different encodings and use of "ret lr" is
; preferred to hint that it is returning to the caller.
;

        IF "$ValidLabel" != ""
            b       $ValidLabel                 ; jump to valid target
        ELIF "$ValidTarget" != "lr"
            br      $ValidTarget                ; jump to valid target
        ELSE
            ret     lr                          ; return
        ENDIF

2 ; Misaligned
        ;
        ; Code on ARM64 should always be 16 byte aligned if address taken.
        ;
        ; TODO:  Compress bitmap format to 1 bit per address on ARM64?
        ; TODO:  We are seeing 8-byte aligned code now. (1/22/14)
        ;
        and     x16, x16, #0xfffffffffffffffe   ; force low bit of shift to 0
        lsr     x17, x17, x16                   ; shift bitmap chunk down
        tbz     x17, #0, $FailLabel             ; invalid if the low bit was clear
        tbnz    x17, #1, %B1                    ; valid if upper bit was set as well
        MEND


        ;
        ; Macro to acquire a spin lock at address $Reg + $Offset. Clobbers {r0-r2}
        ;

; ARM64_WORKITEM : should we use acquire/release semantics instead of DMB?

        MACRO
        ACQUIRE_SPIN_LOCK $Reg, $Offset

        mov     x0, #1                                  ; we want to exchange with a 1
        dmb                                             ; memory barrier ahead of the loop
1
        ldxr    x1, [$Reg, $Offset]                     ; load the new value
        stxr    x2, x0, [$Reg, $Offset]                 ; attempt to store the 1
        cbnz    x2, %B1                                 ; did we succeed before someone else did?
        cbz     x1, %F3                                 ; was the lock previously owned? if not, we're done
        yield                                           ; yield execution
        b       %B1                                     ; and try again
3
        dmb

        MEND


        ;
        ; Macro to release a spin lock at address $Reg + $Offset.
        ;

; ARM64_WORKITEM : should we use acquire/release semantics instead of DMB?

        MACRO
        RELEASE_SPIN_LOCK $Reg, $Offset

        dmb
        str     xzr, [$Reg, $Offset]                    ; store 0

        MEND


        ;
        ; Macro to increment a 64-bit statistic.
        ;

        MACRO
        INCREMENT_STAT $AddrReg, $Temp1, $Temp2

1       ldxr    $Temp1, [$AddrReg]                      ; load current value
        add     $Temp1, $Temp1, #1                      ; increment
        stxr    $Temp2, $Temp1, [$AddrReg]              ; attempt to store
        cbnz    $Temp2, %B1                             ; loop until it works?

        MEND


        ;
        ; Macros to enable/disable interrupts.
        ;
        
        MACRO
        ENABLE_INTERRUPTS
        msr     DAIFClr, #2                             ; enable interrupts
        MEND

        MACRO
        DISABLE_INTERRUPTS
        msr     DAIFSet, #2                             ; disable interrupts
        MEND


        ;
        ; Macros to read/write the current IRQL
        ;
        ; N.B. These macros do not do hardware and software IRQL processing.
        ;

        MACRO
        GET_IRQL $Irql
        ldrb    $Irql, [x18, #PcCurrentIrql]            ; read IRQL
        MEND

        MACRO
        RAISE_IRQL $Reg, $NewIrql

#if DBG

        GET_IRQL $Reg                                   ; get old IRQL
        cmp      $Reg, #$NewIrql                        ; check if old IRQL is lower or same
        bls      %F1                                    ; if LS, valid transition

        EMIT_BREAKPOINT                                 ; break
1
#endif

        mov     $Reg, #$NewIrql                         ; get new IRQL
        strb    $Reg, [x18, #PcCurrentIrql]             ; update IRQL
        MEND


        ;
        ; Macros to output special undefined opcodes that indicate breakpoints
        ; and debug services.
        ;

        MACRO
        EMIT_BREAKPOINT
        brk     #ARM64_BREAKPOINT
        MEND


        MACRO
        EMIT_DEBUG_SERVICE
        brk     #ARM64_DEBUG_SERVICE
        MEND

        MACRO
        FASTFAIL $FastFailCode
        mov     x0, $FastFailCode
        brk     #ARM64_FASTFAIL
        MEND


        ;
        ; Macro to generate an exception frame; this is intended to
        ; be used within the prolog of a function.
        ;

        MACRO
        GENERATE_EXCEPTION_FRAME
        PROLOG_SAVE_REG_PAIR x19, x20, #-96!
        PROLOG_SAVE_REG_PAIR x21, x22, #16
        PROLOG_SAVE_REG_PAIR x23, x24, #32
        PROLOG_SAVE_REG_PAIR x25, x26, #48
        PROLOG_SAVE_REG_PAIR x27, x28, #64
        PROLOG_SAVE_REG_PAIR fp, lr, #80
        MEND


        ;
        ; Macro to restore from an exception frame; this is intended to
        ; be used within the epilog of a function.
        ;

        MACRO
        RESTORE_EXCEPTION_STATE
        EPILOG_RESTORE_REG_PAIR fp, lr, #80
        EPILOG_RESTORE_REG_PAIR x27, x28, #64
        EPILOG_RESTORE_REG_PAIR x25, x26, #48
        EPILOG_RESTORE_REG_PAIR x23, x24, #32
        EPILOG_RESTORE_REG_PAIR x21, x22, #16
        EPILOG_RESTORE_REG_PAIR x19, x20, #96!
        MEND


        ;
        ; Macro to ensure that any eret is followed by barriers to
        ; prevent speculation
        ;
        MACRO
        ERET_FIX
        eret
        dsb sy
        isb sy
        MEND

#define eret ERET_FIX


        ;
        ; Given the Address the EC code range bitmap to determine if it is EC or X64.
        ;
        ; Sets the Zero Flag for X64 targets, clear the Zero Flag for EC targets.
        ;
        ; T1 T2 are scratch registers provided by the caller.
        ;

        MACRO
        EC_BITMAP_LOOKUP $xAddress, $xT1, $wT1, $xT2

        ldr    $xT1, [x18, #TePeb]      ; PEB
        lsr    $xT2, $xAddress, #15     ; each byte of bitmap indexes 8*4K = 2^15 byte span
        ldr    $xT1, [$xT1, #PeEcCodeBitMap]
        ldrb   $wT1, [$xT1, $xT2]       ; load the bitmap byte for the 8*4K span
        ubfx   $xT2, $xAddress, #12, #3 ; index to the 4K page within the 8*4K span
        lsr    $xT1, $xT1, $xT2
        tst    $xT1, #1                 ; test the specific page

        MEND

        ;
        ; Convert NZCV flags in the $Nzcv register into equivalent
        ; AMD64 EFLAGS in the $Eflags register. The source register
        ; is consumed as part of this operation.
        ;

        MACRO
        CONVERT_NZCV_TO_EFLAGS $Eflags, $Nzcv

        mov     $Eflags, #0x0202                ; start with IF=1 and hardcoded bit 1 set

        lsr     $Nzcv, $Nzcv, #21               ; move SS (21) flag to bottom
        bfi     $Eflags, $Nzcv, #8, #1          ; insert at bit 8 (T)
        lsr     $Nzcv, $Nzcv, #7                ; move V (28) flag to bottom
        bfi     $Eflags, $Nzcv, #11, #1         ; insert at bit 11 (O)
        lsr     $Nzcv, $Nzcv, #1                ; move C (29) flag to bottom
        bfi     $Eflags, $Nzcv, #0, #1          ; insert at bit 0 (C)
        lsr     $Nzcv, $Nzcv, #1                ; move Z:N (30:31) flags to bottom
        bfi     $Eflags, $Nzcv, #6, #2          ; insert at bits 6:7 (Z:S)

        MEND

        ;
        ; Convert control and status flags in FPCR and FPSR formats
        ; (in $Fpcr and $Fpsr registers) into a combined AMD64 MXCSR
        ; register. The two source registers are consumed as part of
        ; this operation.
        ;

        MACRO
        CONVERT_FPCR_FPSR_TO_MXCSR $Mxcsr, $Fpcr, $Fpsr

        ;
        ; Status flags map 1:1, if set indicates an exception occured.
        ;

        and     $Mxcsr, $Fpsr, #1               ; MXCSR.IE[0] = FPSR.IOC[0]
        lsr     $Fpsr, $Fpsr, #1                ; remove low bit
        bfi     $Mxcsr, $Fpsr, #2, #4           ; MXCSR.ZE/OE/UE/PE[2:6] = FPSR.DZC/OFC/UFC/IXC[1:4]
        lsr     $Fpsr, $Fpsr, #6                ; remove those bits and the next 2
        bfi     $Mxcsr, $Fpsr, #1, #1           ; MXCSR.DE[1] = FPSR.IDC[7]

        ;
        ; Exception enable bit map 1:1, however on X64 set means mask (disable)
        ; while on ARM64 set means enable the exception.  Thus the bit inversion.
        ;

        lsr     $Fpcr, $Fpcr, #8                ; skip low 8 bits
        eor     $Fpcr, $Fpcr, #0xff             ; invert the masking bits
        bfi     $Mxcsr, $Fpcr, #7, #1           ; MXCSR.IM[7] = ~FPCR.IOE[8]
        lsr     $Fpcr, $Fpcr, #1                ; remove low bit
        bfi     $Mxcsr, $Fpcr, #9, #4           ; MXCSR.ZM/OM/UM/PM[9:12] = ~FPCR.DZE/OFE/UFE/IXE[9:12]
        lsr     $Fpcr, $Fpcr, #6                ; remove those bits and the next 2
        bfi     $Mxcsr, $Fpcr, #8, #1           ; MXCSR.DM[8] = ~FPCR.IDE[15]

        ;
        ; Denormals Are Zeros has not direct mapping on ARM64, use the FZ16 bit
        ; since half-precision floats do not exist in SSE.
        ;

        lsr     $Fpcr, $Fpcr, #4                ; move bit 19 down low
        bfi     $Mxcsr, $Fpcr, #6, #1           ; MXCSR.DAZ[6] = FPCR.FZ[19]

        ;
        ; Rounding modes are the same on X64 and ARM64 except bit swapped in representation.
        ; X64: 00=nearest 01=down 10=up 11=truncate
        ; A64: 00=nearest 10=down 01=up 11=truncate
        ;

        lsr     $Fpcr, $Fpcr, #3                ; move bit 22 down low
        bfi     $Mxcsr, $Fpcr, #14, #1          ; MXCSR.RC[13:14](upper) = FPCR.RMODE[22:23](lower)
        lsr     $Fpcr, $Fpcr, #1                ; get upper bit of rmode in LSB
        bfi     $Mxcsr, $Fpcr, #13, #1          ; MXCSR.RC[13:14](lower) = FPCR.RMODE[22:23](upper)

        ;
        ; Flush To Zero bit maps 1:1
        ;

        lsr     $Fpcr, $Fpcr, #1                ; shift bit 24 down low
        bfi     $Mxcsr, $Fpcr, #15, #1          ; MXCSR.FZ = FPCR.FZ

        MEND
