ntt_dilithium_123_456_78_opt_speed.s

///
/// Copyright (c) 2022 Arm Limited
/// Copyright (c) 2022 Hanno Becker
/// Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer
/// SPDX-License-Identifier: MIT
///
/// Permission is hereby granted, free of charge, to any person obtaining a copy
/// of this software and associated documentation files (the "Software"), to deal
/// in the Software without restriction, including without limitation the rights
/// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
/// copies of the Software, and to permit persons to whom the Software is
/// furnished to do so, subject to the following conditions:
///
/// The above copyright notice and this permission notice shall be included in all
/// copies or substantial portions of the Software.
///
/// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
/// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
/// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
/// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
/// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
/// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
/// SOFTWARE.
///

.data
.p2align 4
roots:
#include "ntt_dilithium_123_456_78_twiddles.s"
.text

// Barrett multiplication
.macro mulmod dst, src, const, const_twisted
        vmul.s32       \dst,  \src, \const
        vqrdmulh.s32   \src,  \src, \const_twisted
        vmla.s32       \dst,  \src, modulus
.endm

.macro ct_butterfly a, b, root, root_twisted
        mulmod tmp, \b, \root, \root_twisted
        vsub.u32       \b,    \a, tmp
        vadd.u32       \a,    \a, tmp
.endm

.macro qsave loc, a       // slothy:no-unfold
        vstrw.32 \a, [sp, #\loc\()]
.endm
.macro qrestore a, loc    // slothy:no-unfold
        vldrw.32 \a, [sp, #\loc\()]
.endm
.macro restored a, b, loc // slothy:no-unfold
        ldrd \a, \b, [sp, #\loc\()]
.endm
.macro saved loc, a, b    // slothy:no-unfold
        strd \a, \b, [sp, #\loc\()]
.endm
.macro restore a, loc     // slothy:no-unfold
        ldr \a, [sp, #\loc\()]
.endm
.macro save loc, a        // slothy:no-unfold
        str \a, [sp, #\loc\()]
.endm

// Aligns stack =0 mod 16
.macro align_stack_do // slothy:no-unfold
        mov r11, sp
        and r12, r11, #0xC   // 8 of ==8 mod 16, 0 otherwise
        sub sp, sp, r12      // Align stack to 16 byte
        sub sp, sp, #16
        str r12, [sp]
.endm

// Reverts initial stack correction
.macro align_stack_undo // slothy:no-unfold
        ldr r12, [sp]
        add sp, sp, #16
        add sp, sp, r12
.endm

#define STACK_SIZE (5*16+8)    // +8 is for alignment
#define QSTACK4 (0*16)
#define QSTACK5 (1*16)
#define QSTACK6 (2*16)

#define ROOT0_STACK (3*16)
#define ROOT1_STACK (3*16 + 8)
#define ROOT4_STACK (4*16)
#define RPTR_STACK  (4*16 + 8)

.align 4
roots_addr: .word roots
.syntax unified
.type ntt_dilithium_123_456_78_opt_speed, %function
.global ntt_dilithium_123_456_78_opt_speed
ntt_dilithium_123_456_78_opt_speed:

        push {r4-r11,lr}
        // Save MVE vector registers
        vpush {d8-d15}

        align_stack_do
        sub sp, sp, #STACK_SIZE

        modulus .req r12
        r_ptr   .req r11

        .equ modulus_const, -8380417
        movw modulus, #:lower16:modulus_const
        movt modulus, #:upper16:modulus_const
        ldr  r_ptr, roots_addr

        in           .req r0
        in_low       .req in
        in_high      .req r1

        add in_high, in, #(4*128)

        root2    .req r2
        root2_tw .req r3
        root3    .req r4
        root3_tw .req r5
        root5    .req r6
        root5_tw .req r7
        root6    .req r8
        root6_tw .req r9

        data0 .req q0
        data1 .req q1
        data2 .req q2
        data3 .req q3
        data4 .req q1
        data5 .req q2
        data6 .req q3
        data7 .req q4

        tmp .req q7

        // Layers 1-3

        rtmp    .req root6
        rtmp_tw .req root6_tw

        ldrd rtmp, rtmp_tw, [r_ptr], #(7*8)
        saved ROOT0_STACK, rtmp, rtmp_tw
        ldrd rtmp, rtmp_tw, [r_ptr, #(1*8 - 7*8)]
        saved ROOT1_STACK, rtmp, rtmp_tw
        ldrd root2, root2_tw, [r_ptr, #(2*8 - 7*8)]
        ldrd root3, root3_tw, [r_ptr, #(3*8 - 7*8)]
        ldrd rtmp, rtmp_tw, [r_ptr, #(4*8 - 7*8)]
        saved ROOT4_STACK, rtmp, rtmp_tw
        ldrd root5, root5_tw, [r_ptr, #(5*8 - 7*8)]
        ldrd root6, root6_tw, [r_ptr, #(6*8 - 7*8)]
        save RPTR_STACK, r_ptr

        .unreq rtmp
        .unreq rtmp_tw
        rtmp    .req r10
        rtmp_tw .req r11

        mov lr, #8
        .p2align 2
        restored r11, r10, ROOT0_STACK
        vldrw.32 q4, [in_high, #128]
        vmul.s32 q6, q4, r11
        vldrw.32 q7, [in_high]
        vqrdmulh.s32 q2, q4, r10
        vldrw.32 q1, [in_low, #128]
        vmla.s32 q6, q2, modulus
        vldrw.32 q2, [in_low, #256]
        vmul.s32 q3, q7, r11
        vsub.u32 q5, q1, q6
        vqrdmulh.s32 q7, q7, r10
        vadd.u32 q4, q1, q6
        vmla.s32 q3, q7, modulus
        vldrw.32 q7, [in_high, #256]
        vmul.s32 q6, q7, r11
        qsave QSTACK5, q5
        vqrdmulh.s32 q0, q7, r10
        vldrw.32 q7, [in_low, #384]
        vmla.s32 q6, q0, modulus
        vldrw.32 q0, [in_high, #384]
        vmul.s32 q1, q0, r11
        vsub.u32 q5, q2, q6
        vqrdmulh.s32 q0, q0, r10
        vadd.u32 q2, q2, q6
        vmla.s32 q1, q0, modulus
        restored r10, r11, ROOT1_STACK
        qsave QSTACK6, q5
        vmul.s32 q0, q2, r10
        vsub.u32 q5, q7, q1
        vqrdmulh.s32 q2, q2, r11
        vldrw.32 q6, [in_low]
        vmla.s32 q0, q2, modulus
        vadd.u32 q2, q7, q1
        vmul.s32 q1, q2, r10
        vsub.u32 q7, q6, q3
        vqrdmulh.s32 q2, q2, r11
        vadd.u32 q6, q6, q3
        vmla.s32 q1, q2, modulus
        qsave QSTACK4, q7
        vadd.u32 q3, q4, q1
        vmul.s32 q2, q3, root2
        vsub.u32 q7, q6, q0
        sub lr, lr, #1
.p2align 2
layer123_loop:
        vqrdmulh.s32 q3, q3, root2_tw          // *....................................................................................
        vsub.u32 q4, q4, q1                    // ...*.................................................................................
        vmul.s32 q1, q4, root3                 // ....*................................................................................
        restored r11, r10, ROOT4_STACK         // ...........*.........................................................................
        vmla.s32 q2, q3, modulus               // ..*..................................................................................
        vadd.u32 q3, q6, q0                    // .*...................................................................................
        qrestore q6, QSTACK6                   // ............*........................................................................
        vqrdmulh.s32 q4, q4, root3_tw          // ......*..............................................................................
        vadd.u32 q0, q3, q2                    // .......*.............................................................................
        vmla.s32 q1, q4, modulus               // ........*............................................................................
        vstrw.u32 q0, [in_low] , #16           // .........*...........................................................................
        vsub.u32 q0, q3, q2                    // .....*...............................................................................
        vqrdmulh.s32 q3, q6, r10               // ...............*.....................................................................
        vstrw.u32 q0, [in_low, #112]           // ................*....................................................................
        vqrdmulh.s32 q0, q5, r10               // .....................*...............................................................
        vsub.u32 q2, q7, q1                    // ..............*......................................................................
        vmul.s32 q4, q5, r11                   // ...................*.................................................................
        vstrw.u32 q2, [in_low, #368]           // ...............................*.....................................................
        vmla.s32 q4, q0, modulus               // .......................*.............................................................
        vadd.u32 q2, q7, q1                    // ..........*..........................................................................
        vmul.s32 q0, q6, r11                   // .............*.......................................................................
        qrestore q6, QSTACK5                   // ........................*............................................................
        vmla.s32 q0, q3, modulus               // .................*...................................................................
        vsub.u32 q1, q6, q4                    // .............................*.......................................................
        vmul.s32 q7, q1, root6                 // .................................*...................................................
        qrestore q5, QSTACK4                   // ....................*................................................................
        vqrdmulh.s32 q1, q1, root6_tw          // ...................................*.................................................
        vsub.u32 q3, q5, q0                    // ......................*..............................................................
        vmla.s32 q7, q1, modulus               // .....................................*...............................................
        vadd.u32 q4, q6, q4                    // .........................*...........................................................
        vqrdmulh.s32 q6, q4, root5_tw          // ............................*........................................................
        vsub.u32 q1, q3, q7                    // .......................................*.............................................
        vstrw.u32 q1, [in_high, #384]          // ........................................*............................................
        vmul.s32 q1, q4, root5                 // ..........................*..........................................................
        restored r10, r11, ROOT0_STACK         // ...........................................*.........................................
        vmla.s32 q1, q6, modulus               // ..............................*......................................................
        vldrw.32 q4, [in_high, #272]           // ........................................................*............................
        vadd.u32 q6, q5, q0                    // ...........................*.........................................................
        vqrdmulh.s32 q5, q4, r11               // ...........................................................*.........................
        vsub.u32 q0, q6, q1                    // ................................*....................................................
        vstrw.u32 q0, [in_high, #128]          // ......................................*..............................................
        vmul.s32 q4, q4, r10                   // .........................................................*...........................
        vadd.u32 q1, q6, q1                    // ..................................*..................................................
        vldrw.32 q6, [in_low, #256]            // ..................................................*..................................
        vmla.s32 q4, q5, modulus               // .............................................................*.......................
        vldrw.32 q0, [in_high, #400]           // ..............................................................*......................
        vadd.u32 q5, q3, q7                    // .........................................*...........................................
        vldrw.32 q7, [in_high, #144]           // ............................................*........................................
        vqrdmulh.s32 q3, q0, r11               // .................................................................*...................
        vstrw.u32 q2, [in_low, #240]           // ..................*..................................................................
        vmul.s32 q2, q7, r10                   // .............................................*.......................................
        vstrw.u32 q1, [in_high] , #16          // ....................................*................................................
        vqrdmulh.s32 q1, q7, r11               // ...............................................*.....................................
        vadd.u32 q7, q6, q4                    // ..................................................................*..................
        vstrw.u32 q5, [in_high, #240]          // ..........................................*..........................................
        vmla.s32 q2, q1, modulus               // .................................................*...................................
        vsub.u32 q6, q6, q4                    // ................................................................*....................
        vldrw.32 q4, [in_high]                 // ..............................................*......................................
        vmul.s32 q0, q0, r10                   // ...............................................................*.....................
        vldrw.32 q5, [in_low, #128]            // ................................................*....................................
        vmla.s32 q0, q3, modulus               // ...................................................................*.................
        vsub.u32 q1, q5, q2                    // ....................................................*................................
        qsave QSTACK5, q1                      // ..........................................................*..........................
        vqrdmulh.s32 q3, q4, r11               // .....................................................*...............................
        qsave QSTACK6, q6                      // .....................................................................*...............
        vmul.s32 q6, q4, r10                   // ...................................................*.................................
        restored r11, r10, ROOT1_STACK         // ....................................................................*................
        vadd.u32 q4, q5, q2                    // ......................................................*..............................
        vldrw.32 q1, [in_low, #384]            // ............................................................*........................
        vmla.s32 q6, q3, modulus               // .......................................................*.............................
        vadd.u32 q3, q1, q0                    // ...........................................................................*.........
        vqrdmulh.s32 q2, q3, r10               // ..............................................................................*......
        vsub.u32 q5, q1, q0                    // .......................................................................*.............
        vmul.s32 q1, q3, r11                   // ............................................................................*........
        vldrw.32 q0, [in_low]                  // .........................................................................*...........
        vmla.s32 q1, q2, modulus               // ................................................................................*....
        vsub.u32 q3, q0, q6                    // .............................................................................*.......
        vqrdmulh.s32 q2, q7, r10               // ........................................................................*............
        vadd.u32 q6, q0, q6                    // ...............................................................................*.....
        vmul.s32 q0, q7, r11                   // ......................................................................*..............
        qsave QSTACK4, q3                      // .................................................................................*...
        vmla.s32 q0, q2, modulus               // ..........................................................................*..........
        vadd.u32 q3, q4, q1                    // ..................................................................................*..
        vmul.s32 q2, q3, root2                 // ...................................................................................*.
        vsub.u32 q7, q6, q0                    // ....................................................................................*
        
        // original source code
        // vqrdmulh.s32 q3, q3, root2_tw       // *....................................................................................
        // vadd.u32 q6, q6, q0                 // .....*...............................................................................
        // vmla.s32 q2, q3, modulus            // ....*................................................................................
        // vsub.u32 q4, q4, q1                 // .*...................................................................................
        // vmul.s32 q0, q4, root3              // ..*..................................................................................
        // vsub.u32 q3, q6, q2                 // ...........*.........................................................................
        // vqrdmulh.s32 q1, q4, root3_tw       // .......*.............................................................................
        // vadd.u32 q6, q6, q2                 // ........*............................................................................
        // vmla.s32 q0, q1, modulus            // .........*...........................................................................
        // vstrw.u32 q6, [in_low] , #16        // ..........*..........................................................................
        // vadd.u32 q4, q7, q0                 // ...................*.................................................................
        // restored r10, r11, ROOT4_STACK      // ...*.................................................................................
        // qrestore q1, QSTACK6                // ......*..............................................................................
        // vmul.s32 q2, q1, r10                // ....................*................................................................
        // vsub.u32 q6, q7, q0                 // ...............*.....................................................................
        // vqrdmulh.s32 q7, q1, r11            // ............*........................................................................
        // vstrw.u32 q3, [in_low, #112]        // .............*.......................................................................
        // vmla.s32 q2, q7, modulus            // ......................*..............................................................
        // vstrw.u32 q4, [in_low, #240]        // .................................................*...................................
        // vmul.s32 q7, q5, r10                // ................*....................................................................
        // qrestore q3, QSTACK4                // .........................*...........................................................
        // vqrdmulh.s32 q0, q5, r11            // ..............*......................................................................
        // vsub.u32 q1, q3, q2                 // ...........................*.........................................................
        // vmla.s32 q7, q0, modulus            // ..................*..................................................................
        // qrestore q5, QSTACK5                // .....................*...............................................................
        // vadd.u32 q4, q5, q7                 // .............................*.......................................................
        // vmul.s32 q0, q4, root5              // .................................*...................................................
        // vadd.u32 q3, q3, q2                 // .....................................*...............................................
        // vqrdmulh.s32 q4, q4, root5_tw       // ..............................*......................................................
        // vsub.u32 q7, q5, q7                 // .......................*.............................................................
        // vmla.s32 q0, q4, modulus            // ...................................*.................................................
        // vstrw.u32 q6, [in_low, #368]        // .................*...................................................................
        // vsub.u32 q4, q3, q0                 // .......................................*.............................................
        // vmul.s32 q2, q7, root6              // ........................*............................................................
        // vadd.u32 q3, q3, q0                 // ..........................................*..........................................
        // vqrdmulh.s32 q7, q7, root6_tw       // ..........................*..........................................................
        // vstrw.u32 q3, [in_high] , #16       // ...................................................*.................................
        // vmla.s32 q2, q7, modulus            // ............................*........................................................
        // vstrw.u32 q4, [in_high, #112]       // ........................................*............................................
        // vsub.u32 q3, q1, q2                 // ...............................*.....................................................
        // vstrw.u32 q3, [in_high, #368]       // ................................*....................................................
        // vadd.u32 q3, q1, q2                 // ..............................................*......................................
        // vstrw.u32 q3, [in_high, #240]       // ......................................................*..............................
        // restored r11, r10, ROOT0_STACK      // ..................................*..................................................
        // vldrw.32 q4, [in_high, #128]        // ...............................................*.....................................
        // vmul.s32 q6, q4, r11                // ..................................................*..................................
        // vldrw.32 q7, [in_high]              // .........................................................*...........................
        // vqrdmulh.s32 q2, q4, r10            // ....................................................*................................
        // vldrw.32 q1, [in_low, #128]         // ...........................................................*.........................
        // vmla.s32 q6, q2, modulus            // .......................................................*.............................
        // vldrw.32 q2, [in_low, #256]         // ...........................................*.........................................
        // vmul.s32 q3, q7, r11                // .................................................................*...................
        // vsub.u32 q5, q1, q6                 // .............................................................*.......................
        // vqrdmulh.s32 q7, q7, r10            // ...............................................................*.....................
        // vadd.u32 q4, q1, q6                 // ...................................................................*.................
        // vmla.s32 q3, q7, modulus            // .....................................................................*...............
        // vldrw.32 q7, [in_high, #256]        // ....................................*................................................
        // vmul.s32 q6, q7, r11                // .........................................*...........................................
        // qsave QSTACK5, q5                   // ..............................................................*......................
        // vqrdmulh.s32 q0, q7, r10            // ......................................*..............................................
        // vldrw.32 q7, [in_low, #384]         // ....................................................................*................
        // vmla.s32 q6, q0, modulus            // ............................................*........................................
        // vldrw.32 q0, [in_high, #384]        // .............................................*.......................................
        // vmul.s32 q1, q0, r11                // ..........................................................*..........................
        // vsub.u32 q5, q2, q6                 // ........................................................*............................
        // vqrdmulh.s32 q0, q0, r10            // ................................................*....................................
        // vadd.u32 q2, q2, q6                 // .....................................................*...............................
        // vmla.s32 q1, q0, modulus            // ............................................................*........................
        // restored r10, r11, ROOT1_STACK      // ..................................................................*..................
        // qsave QSTACK6, q5                   // ................................................................*....................
        // vmul.s32 q0, q2, r10                // ...............................................................................*.....
        // vsub.u32 q5, q7, q1                 // ........................................................................*............
        // vqrdmulh.s32 q2, q2, r11            // .............................................................................*.......
        // vldrw.32 q6, [in_low]               // ..........................................................................*..........
        // vmla.s32 q0, q2, modulus            // .................................................................................*...
        // vadd.u32 q2, q7, q1                 // ......................................................................*..............
        // vmul.s32 q1, q2, r10                // .........................................................................*...........
        // vsub.u32 q7, q6, q3                 // ............................................................................*........
        // vqrdmulh.s32 q2, q2, r11            // .......................................................................*.............
        // vadd.u32 q6, q6, q3                 // ..............................................................................*......
        // vmla.s32 q1, q2, modulus            // ...........................................................................*.........
        // qsave QSTACK4, q7                   // ................................................................................*....
        // vadd.u32 q3, q4, q1                 // ..................................................................................*..
        // vmul.s32 q2, q3, root2              // ...................................................................................*.
        // vsub.u32 q7, q6, q0                 // ....................................................................................*
        
        le lr, layer123_loop
layer123_loop_end: // end of loop kernel
        vqrdmulh.s32 q3, q3, root2_tw
        vadd.u32 q6, q6, q0
        vmla.s32 q2, q3, modulus
        vsub.u32 q4, q4, q1
        vmul.s32 q0, q4, root3
        vsub.u32 q3, q6, q2
        vqrdmulh.s32 q1, q4, root3_tw
        vadd.u32 q6, q6, q2
        vmla.s32 q0, q1, modulus
        vstrw.u32 q6, [in_low] , #16
        vadd.u32 q4, q7, q0
        restored r10, r11, ROOT4_STACK
        qrestore q1, QSTACK6
        vmul.s32 q2, q1, r10
        vsub.u32 q6, q7, q0
        vqrdmulh.s32 q7, q1, r11
        vstrw.u32 q3, [in_low, #112]
        vmla.s32 q2, q7, modulus
        vstrw.u32 q4, [in_low, #240]
        vmul.s32 q7, q5, r10
        qrestore q3, QSTACK4
        vqrdmulh.s32 q0, q5, r11
        vsub.u32 q1, q3, q2
        vmla.s32 q7, q0, modulus
        qrestore q5, QSTACK5
        vadd.u32 q4, q5, q7
        vmul.s32 q0, q4, root5
        vadd.u32 q3, q3, q2
        vqrdmulh.s32 q4, q4, root5_tw
        vsub.u32 q7, q5, q7
        vmla.s32 q0, q4, modulus
        vstrw.u32 q6, [in_low, #368]
        vsub.u32 q4, q3, q0
        vmul.s32 q2, q7, root6
        vadd.u32 q3, q3, q0
        vqrdmulh.s32 q7, q7, root6_tw
        vstrw.u32 q3, [in_high] , #16
        vmla.s32 q2, q7, modulus
        vstrw.u32 q4, [in_high, #112]
        vsub.u32 q3, q1, q2
        vstrw.u32 q3, [in_high, #368]
        vadd.u32 q3, q1, q2
        vstrw.u32 q3, [in_high, #240]

        .unreq in_high
        .unreq in_low

        sub in, in, #(128)
        restore r_ptr, RPTR_STACK

        // Layers 4,5,6

        .unreq rtmp
        .unreq rtmp_tw
        rtmp    .req r3
        rtmp_tw .req r4

        mov lr, #8
        .p2align 2
        ldrd r2, r3, [r_ptr] , #56
        vldrw.32 q0, [in, #64]
        vmul.s32 q3, q0, r2
        vldrw.32 q6, [in, #96]
        vqrdmulh.s32 q2, q6, r3
        vldrw.32 q1, [in, #32]
        vmul.s32 q6, q6, r2
        ldrd r8, r5, [r_ptr, #-48]
        vmla.s32 q6, q2, modulus
        vldrw.32 q2, [in]
        vqrdmulh.s32 q4, q0, r3
        vadd.u32 q0, q1, q6
        vmla.s32 q3, q4, modulus
        vsub.u32 q1, q1, q6
        qsave QSTACK6, q1
        vsub.u32 q5, q2, q3
        vqrdmulh.s32 q7, q0, r5
        qsave QSTACK4, q5
        vmul.s32 q1, q0, r8
        vadd.u32 q4, q2, q3
        vmla.s32 q1, q7, modulus
        vldrw.32 q3, [in, #112]
        vsub.u32 q2, q4, q1
        vqrdmulh.s32 q7, q3, r3
        vadd.u32 q1, q4, q1
        vmul.s32 q5, q3, r2
        vldrw.32 q4, [in, #48]
        vmla.s32 q5, q7, modulus
        vldrw.32 q7, [in, #80]
        vmul.s32 q6, q7, r2
        vsub.u32 q3, q4, q5
        vqrdmulh.s32 q0, q7, r3
        vadd.u32 q5, q4, q5
        vmla.s32 q6, q0, modulus
        vldrw.32 q4, [in, #16]
        vqrdmulh.s32 q0, q5, r5
        vadd.u32 q7, q4, q6
        vmul.s32 q5, q5, r8
        vsub.u32 q6, q4, q6
        vmla.s32 q5, q0, modulus
        qsave QSTACK5, q6
        vadd.u32 q4, q7, q5
        ldrd r7, r6, [r_ptr, #-32]
        vsub.u32 q6, q7, q5
        sub lr, lr, #1
.p2align 2
layer456_loop:
        vqrdmulh.s32 q5, q6, r6              // *........................................................................................
        ldrd r6, r5, [r_ptr, #-24]           // ...*.....................................................................................
        vmul.s32 q0, q6, r7                  // ..*......................................................................................
        ldrd r2, r3, [r_ptr, #-40]           // .*.......................................................................................
        vmla.s32 q0, q5, modulus             // ......*..................................................................................
        qrestore q5, QSTACK6                 // .............*...........................................................................
        vqrdmulh.s32 q6, q4, r3              // ................*........................................................................
        vsub.u32 q7, q2, q0                  // ...........*.............................................................................
        vmul.s32 q4, q4, r2                  // ....*....................................................................................
        vstrw.u32 q7, [in, #48]              // ...............*.........................................................................
        vmla.s32 q4, q6, modulus             // ..................*......................................................................
        vadd.u32 q7, q2, q0                  // ........*................................................................................
        vmul.s32 q2, q3, r6                  // ..........*..............................................................................
        vadd.u32 q0, q1, q4                  // .......................*.................................................................
        vqrdmulh.s32 q6, q3, r5              // ............*............................................................................
        vstrw.u32 q0, [in] , #128            // ......................................*..................................................
        vmul.s32 q0, q5, r6                  // ......................*..................................................................
        qrestore q3, QSTACK5                 // .......*.................................................................................
        vmla.s32 q2, q6, modulus             // ..............*..........................................................................
        ldrd r9, r8, [r_ptr, #-16]           // ...............................*.........................................................
        vqrdmulh.s32 q5, q5, r5              // ........................*................................................................
        vadd.u32 q6, q3, q2                  // ...................*.....................................................................
        vmla.s32 q0, q5, modulus             // ..........................*..............................................................
        vsub.u32 q1, q1, q4                  // .....................*...................................................................
        vmul.s32 q4, q6, r9                  // ...................................*.....................................................
        vstrw.u32 q1, [in, #-112]            // ...........................*.............................................................
        vqrdmulh.s32 q5, q6, r8              // .....................................*...................................................
        vsub.u32 q3, q3, q2                  // .................*.......................................................................
        vmla.s32 q4, q5, modulus             // .......................................*.................................................
        qrestore q6, QSTACK4                 // .........................*...............................................................
        vadd.u32 q1, q6, q0                  // ....................................*....................................................
        vldrw.32 q5, [in, #96]               // ................................................*........................................
        vsub.u32 q2, q1, q4                  // ...........................................*.............................................
        ldrd r6, r7, [r_ptr, #-8]            // .....*...................................................................................
        vstrw.u32 q2, [in, #-48]             // ............................................*............................................
        vadd.u32 q1, q1, q4                  // .........................................*...............................................
        vmul.s32 q2, q3, r6                  // ............................*............................................................
        ldrd r1, r9, [r_ptr] , #56           // .............................................*...........................................
        vqrdmulh.s32 q4, q3, r7              // ....................*....................................................................
        vstrw.u32 q1, [in, #-64]             // ..........................................*..............................................
        vmul.s32 q3, q5, r1                  // ...................................................*.....................................
        vstrw.u32 q7, [in, #-96]             // .........*...............................................................................
        vmla.s32 q2, q4, modulus             // ..............................*..........................................................
        vsub.u32 q7, q6, q0                  // .............................*...........................................................
        vqrdmulh.s32 q6, q5, r9              // .................................................*.......................................
        vldrw.32 q4, [in, #64]               // ..............................................*..........................................
        vmla.s32 q3, q6, modulus             // .....................................................*...................................
        vsub.u32 q0, q7, q2                  // ..................................*......................................................
        vldrw.32 q5, [in, #48]               // .......................................................................*.................
        vmul.s32 q1, q4, r1                  // ...............................................*.........................................
        vldrw.32 q6, [in, #32]               // ..................................................*......................................
        vadd.u32 q7, q7, q2                  // ................................*........................................................
        vstrw.u32 q7, [in, #-32]             // .................................*.......................................................
        vadd.u32 q7, q6, q3                  // ........................................................*................................
        vqrdmulh.s32 q4, q4, r9              // .......................................................*.................................
        vsub.u32 q2, q6, q3                  // ..........................................................*..............................
        vldrw.32 q6, [in, #112]              // ..................................................................*......................
        vqrdmulh.s32 q3, q6, r9              // ....................................................................*....................
        ldrd r10, r3, [r_ptr, #-48]          // ....................................................*....................................
        vmul.s32 q6, q6, r1                  // ......................................................................*..................
        qsave QSTACK6, q2                    // ...........................................................*.............................
        vmla.s32 q6, q3, modulus             // ........................................................................*................
        vstrw.u32 q0, [in, #-16]             // ........................................*................................................
        vqrdmulh.s32 q2, q7, r3              // .............................................................*...........................
        vsub.u32 q3, q5, q6                  // ...........................................................................*.............
        vmul.s32 q0, q7, r10                 // ...............................................................*.........................
        vadd.u32 q5, q5, q6                  // .............................................................................*...........
        vmla.s32 q0, q2, modulus             // .................................................................*.......................
        vldrw.32 q7, [in, #80]               // .........................................................................*...............
        vmla.s32 q1, q4, modulus             // .........................................................*...............................
        vldrw.32 q4, [in]                    // ......................................................*..................................
        vqrdmulh.s32 q6, q7, r9              // ............................................................................*............
        vadd.u32 q2, q4, q1                  // ................................................................*........................
        vmul.s32 q7, q7, r1                  // ..........................................................................*..............
        vsub.u32 q1, q4, q1                  // ............................................................*............................
        vqrdmulh.s32 q4, q5, r3              // ................................................................................*........
        qsave QSTACK4, q1                    // ..............................................................*..........................
        vmul.s32 q5, q5, r10                 // ..................................................................................*......
        vadd.u32 q1, q2, q0                  // .....................................................................*...................
        vmla.s32 q7, q6, modulus             // ..............................................................................*..........
        vsub.u32 q2, q2, q0                  // ...................................................................*.....................
        vldrw.32 q6, [in, #16]               // ...............................................................................*.........
        vsub.u32 q0, q6, q7                  // ...................................................................................*.....
        vmla.s32 q5, q4, modulus             // ....................................................................................*....
        vadd.u32 q7, q6, q7                  // .................................................................................*.......
        qsave QSTACK5, q0                    // .....................................................................................*...
        vsub.u32 q6, q7, q5                  // ........................................................................................*
        ldrd r7, r6, [r_ptr, #-32]           // .......................................................................................*.
        vadd.u32 q4, q7, q5                  // ......................................................................................*..
        
        // original source code
        // vqrdmulh.s32 q5, q6, r6           // *........................................................................................
        // ldrd r4, r10, [r_ptr, #-40]       // ...*.....................................................................................
        // vmul.s32 q7, q6, r7               // ..*......................................................................................
        // ldrd r9, r1, [r_ptr, #-24]        // .*.......................................................................................
        // vmul.s32 q0, q4, r4               // ........*................................................................................
        // ldrd r4, r3, [r_ptr, #-8]         // .................................*.......................................................
        // vmla.s32 q7, q5, modulus          // ....*....................................................................................
        // qrestore q5, QSTACK5              // .................*.......................................................................
        // vadd.u32 q6, q2, q7               // ...........*.............................................................................
        // vstrw.u32 q6, [in, #32]           // .........................................*...............................................
        // vmul.s32 q6, q3, r9               // ............*............................................................................
        // vsub.u32 q2, q2, q7               // .......*.................................................................................
        // vqrdmulh.s32 q3, q3, r1           // ..............*..........................................................................
        // qrestore q7, QSTACK6              // .....*...................................................................................
        // vmla.s32 q6, q3, modulus          // ..................*......................................................................
        // vstrw.u32 q2, [in, #48]           // .........*...............................................................................
        // vqrdmulh.s32 q3, q4, r10          // ......*..................................................................................
        // vsub.u32 q2, q5, q6               // ...........................*.............................................................
        // vmla.s32 q0, q3, modulus          // ..........*..............................................................................
        // vadd.u32 q5, q5, q6               // .....................*...................................................................
        // vqrdmulh.s32 q3, q2, r3           // ......................................*..................................................
        // vsub.u32 q4, q1, q0               // .......................*.................................................................
        // vmul.s32 q6, q7, r9               // ................*........................................................................
        // vadd.u32 q0, q1, q0               // .............*...........................................................................
        // vqrdmulh.s32 q7, q7, r1           // ....................*....................................................................
        // qrestore q1, QSTACK4              // .............................*...........................................................
        // vmla.s32 q6, q7, modulus          // ......................*..................................................................
        // vstrw.u32 q4, [in, #16]           // .........................*...............................................................
        // vmul.s32 q7, q2, r4               // ....................................*....................................................
        // vsub.u32 q2, q1, q6               // ...........................................*.............................................
        // vmla.s32 q7, q3, modulus          // ..........................................*..............................................
        // ldrd r1, r2, [r_ptr, #-16]        // ...................*.....................................................................
        // vadd.u32 q4, q2, q7               // ...................................................*.....................................
        // vstrw.u32 q4, [in, #96]           // ....................................................*....................................
        // vsub.u32 q7, q2, q7               // ...............................................*.........................................
        // vmul.s32 q2, q5, r1               // ........................*................................................................
        // vadd.u32 q4, q1, q6               // ..............................*..........................................................
        // vqrdmulh.s32 q5, q5, r2           // ..........................*..............................................................
        // vstrw.u32 q0, [in] , #128         // ...............*.........................................................................
        // vmla.s32 q2, q5, modulus          // ............................*............................................................
        // vstrw.u32 q7, [in, #-16]          // ..............................................................*..........................
        // vadd.u32 q1, q4, q2               // ...................................*.....................................................
        // vstrw.u32 q1, [in, #-64]          // .......................................*.................................................
        // vsub.u32 q4, q4, q2               // ................................*........................................................
        // vstrw.u32 q4, [in, #-48]          // ..................................*......................................................
        // ldrd r2, r3, [r_ptr] , #56        // .....................................*...................................................
        // vldrw.32 q0, [in, #64]            // .............................................*...........................................
        // vmul.s32 q3, q0, r2               // .................................................*.......................................
        // vldrw.32 q6, [in, #96]            // ...............................*.........................................................
        // vqrdmulh.s32 q2, q6, r3           // ............................................*............................................
        // vldrw.32 q1, [in, #32]            // ..................................................*......................................
        // vmul.s32 q6, q6, r2               // ........................................*................................................
        // ldrd r8, r5, [r_ptr, #-48]        // ..........................................................*..............................
        // vmla.s32 q6, q2, modulus          // ..............................................*..........................................
        // vldrw.32 q2, [in]                 // ......................................................................*..................
        // vqrdmulh.s32 q4, q0, r3           // ......................................................*..................................
        // vadd.u32 q0, q1, q6               // .....................................................*...................................
        // vmla.s32 q3, q4, modulus          // .....................................................................*...................
        // vsub.u32 q1, q1, q6               // .......................................................*.................................
        // qsave QSTACK6, q1                 // ............................................................*............................
        // vsub.u32 q5, q2, q3               // ..........................................................................*..............
        // vqrdmulh.s32 q7, q0, r5           // ...............................................................*.........................
        // qsave QSTACK4, q5                 // ............................................................................*............
        // vmul.s32 q1, q0, r8               // .................................................................*.......................
        // vadd.u32 q4, q2, q3               // ........................................................................*................
        // vmla.s32 q1, q7, modulus          // ...................................................................*.....................
        // vldrw.32 q3, [in, #112]           // ........................................................*................................
        // vsub.u32 q2, q4, q1               // ................................................................................*........
        // vqrdmulh.s32 q7, q3, r3           // .........................................................*...............................
        // vadd.u32 q1, q4, q1               // ..............................................................................*..........
        // vmul.s32 q5, q3, r2               // ...........................................................*.............................
        // vldrw.32 q4, [in, #48]            // ................................................*........................................
        // vmla.s32 q5, q7, modulus          // .............................................................*...........................
        // vldrw.32 q7, [in, #80]            // ....................................................................*....................
        // vmul.s32 q6, q7, r2               // .........................................................................*...............
        // vsub.u32 q3, q4, q5               // ................................................................*........................
        // vqrdmulh.s32 q0, q7, r3           // .......................................................................*.................
        // vadd.u32 q5, q4, q5               // ..................................................................*......................
        // vmla.s32 q6, q0, modulus          // ...............................................................................*.........
        // vldrw.32 q4, [in, #16]            // .................................................................................*.......
        // vqrdmulh.s32 q0, q5, r5           // ...........................................................................*.............
        // vadd.u32 q7, q4, q6               // ....................................................................................*....
        // vmul.s32 q5, q5, r8               // .............................................................................*...........
        // vsub.u32 q6, q4, q6               // ..................................................................................*......
        // vmla.s32 q5, q0, modulus          // ...................................................................................*.....
        // qsave QSTACK5, q6                 // .....................................................................................*...
        // vadd.u32 q4, q7, q5               // ........................................................................................*
        // ldrd r7, r6, [r_ptr, #-32]        // .......................................................................................*.
        // vsub.u32 q6, q7, q5               // ......................................................................................*..
        
        le lr, layer456_loop
layer456_loop_end:
        vmul.s32 q0, q6, r7                   // ..*.....................................................
        ldrd r8, r3, [r_ptr, #-8]             // .....*..................................................
        vqrdmulh.s32 q6, q6, r6               // *.......................................................
        ldrd r2, r5, [r_ptr, #-24]            // ...*....................................................
        vmla.s32 q0, q6, modulus              // ......*.................................................
        ldrd r6, r7, [r_ptr, #-40]            // .*......................................................
        vqrdmulh.s32 q5, q3, r5               // ............*...........................................
        vadd.u32 q6, q2, q0                   // ........*...............................................
        vmul.s32 q7, q4, r6                   // ....*...................................................
        vsub.u32 q2, q2, q0                   // ...........*............................................
        vmul.s32 q0, q3, r2                   // ..........*.............................................
        vstrw.u32 q2, [in, #48]               // ...............*........................................
        vmla.s32 q0, q5, modulus              // ..............*.........................................
        qrestore q2, QSTACK5                  // .......*................................................
        vqrdmulh.s32 q3, q4, r7               // ................*.......................................
        vsub.u32 q5, q2, q0                   // .................*......................................
        vqrdmulh.s32 q4, q5, r3               // ....................*...................................
        vstrw.u32 q6, [in, #32]               // .........*..............................................
        vmla.s32 q7, q3, modulus              // ..................*.....................................
        qrestore q3, QSTACK6                  // .............*..........................................
        vsub.u32 q6, q1, q7                   // .....................*..................................
        vstrw.u32 q6, [in, #16]               // ...........................*............................
        vadd.u32 q7, q1, q7                   // .......................*................................
        qrestore q6, QSTACK4                  // .........................*..............................
        vqrdmulh.s32 q1, q3, r5               // ........................*...............................
        ldrd r3, r9, [r_ptr, #-16]            // ...............................*........................
        vmul.s32 q3, q3, r2                   // ......................*.................................
        vadd.u32 q2, q2, q0                   // ...................*....................................
        vmla.s32 q3, q1, modulus              // ..........................*.............................
        vstrw.u32 q7, [in] , #128             // ......................................*.................
        vmul.s32 q1, q2, r3                   // ...................................*....................
        vadd.u32 q7, q6, q3                   // ....................................*...................
        vmul.s32 q5, q5, r8                   // ............................*...........................
        vsub.u32 q6, q6, q3                   // .............................*..........................
        vmla.s32 q5, q4, modulus              // ..............................*.........................
        vldrw.32 q0, [r_ptr, #16]             // ...............................................*........
        vadd.u32 q4, q6, q5                   // ................................*.......................
        vstrw.u32 q4, [in, #-32]              // .................................*......................
        vqrdmulh.s32 q4, q2, r9               // .....................................*..................
        vsub.u32 q5, q6, q5                   // ..................................*.....................
        vmla.s32 q1, q4, modulus              // .......................................*................
        vstrw.u32 q5, [in, #-16]              // ........................................*...............
        vsub.u32 q2, q7, q1                   // ...........................................*............
        vstrw.u32 q2, [in, #-48]              // ............................................*...........
        vadd.u32 q1, q7, q1                   // .........................................*..............
        vstrw.u32 q1, [in, #-64]              // ..........................................*.............
        sub in, in, #(4*256)                  // .............................................*..........
        // bubble                             // .......................................................*
        vld40.32 {q3,q4,q5,q6}, [in]          // ................................................*.......
        mov r2, #16                           // ..............................................*.........
        vld41.32 {q3,q4,q5,q6}, [in]          // .................................................*......
        // bubble                             // ......................................................*.
        vld42.32 {q3,q4,q5,q6}, [in]          // ..................................................*.....
        sub lr, r2, #1                        // .....................................................*..
        vld43.32 {q3,q4,q5,q6}, [in]!         // ...................................................*....
        vqrdmulh.s32 q1, q6, q0               // ....................................................*...
        
        // original source code
        // vqrdmulh.s32 q5, q6, r6            // ..*.....................................................
        // ldrd r4, r10, [r_ptr, #-40]        // .....*..................................................
        // vmul.s32 q7, q6, r7                // *.......................................................
        // ldrd r9, r1, [r_ptr, #-24]         // ...*....................................................
        // vmul.s32 q0, q4, r4                // ........*...............................................
        // ldrd r4, r3, [r_ptr, #-8]          // .*......................................................
        // vmla.s32 q7, q5, modulus           // ....*...................................................
        // qrestore q5, QSTACK5               // .............*..........................................
        // vadd.u32 q6, q2, q7                // .......*................................................
        // vstrw.u32 q6, [in, #32]            // .................*......................................
        // vmul.s32 q6, q3, r9                // ..........*.............................................
        // vsub.u32 q2, q2, q7                // .........*..............................................
        // vqrdmulh.s32 q3, q3, r1            // ......*.................................................
        // qrestore q7, QSTACK6               // ...................*....................................
        // vmla.s32 q6, q3, modulus           // ............*...........................................
        // vstrw.u32 q2, [in, #48]            // ...........*............................................
        // vqrdmulh.s32 q3, q4, r10           // ..............*.........................................
        // vsub.u32 q2, q5, q6                // ...............*........................................
        // vmla.s32 q0, q3, modulus           // ..................*.....................................
        // vadd.u32 q5, q5, q6                // ...........................*............................
        // vqrdmulh.s32 q3, q2, r3            // ................*.......................................
        // vsub.u32 q4, q1, q0                // ....................*...................................
        // vmul.s32 q6, q7, r9                // ..........................*.............................
        // vadd.u32 q0, q1, q0                // ......................*.................................
        // vqrdmulh.s32 q7, q7, r1            // ........................*...............................
        // qrestore q1, QSTACK4               // .......................*................................
        // vmla.s32 q6, q7, modulus           // ............................*...........................
        // vstrw.u32 q4, [in, #16]            // .....................*..................................
        // vmul.s32 q7, q2, r4                // ................................*.......................
        // vsub.u32 q2, q1, q6                // .................................*......................
        // vmla.s32 q7, q3, modulus           // ..................................*.....................
        // ldrd r1, r2, [r_ptr, #-16]         // .........................*..............................
        // vadd.u32 q4, q2, q7                // ....................................*...................
        // vstrw.u32 q4, [in, #96]            // .....................................*..................
        // vsub.u32 q7, q2, q7                // .......................................*................
        // vmul.s32 q2, q5, r1                // ..............................*.........................
        // vadd.u32 q4, q1, q6                // ...............................*........................
        // vqrdmulh.s32 q5, q5, r2            // ......................................*.................
        // vstrw.u32 q0, [in] , #128          // .............................*..........................
        // vmla.s32 q2, q5, modulus           // ........................................*...............
        // vstrw.u32 q7, [in, #-16]           // .........................................*..............
        // vadd.u32 q1, q4, q2                // ............................................*...........
        // vstrw.u32 q1, [in, #-64]           // .............................................*..........
        // vsub.u32 q4, q4, q2                // ..........................................*.............
        // vstrw.u32 q4, [in, #-48]           // ...........................................*............
        // sub in, in, #(4*256)               // ..............................................*.........
        // mov lr, #16                        // .................................................*......
        // vldrw.32 q0, [r_ptr, #16]          // ...................................*....................
        // vld40.32 {q3,q4,q5,q6}, [in]       // ................................................*.......
        // vld41.32 {q3,q4,q5,q6}, [in]       // ..................................................*.....
        // vld42.32 {q3,q4,q5,q6}, [in]       // ....................................................*...
        // vld43.32 {q3,q4,q5,q6}, [in]!      // ......................................................*.
        // vqrdmulh.s32 q1, q6, q0            // .......................................................*
        // sub lr, lr, #1                     // .....................................................*..
        // nop                                // ...................................................*....
        // nop                                // ...............................................*........
        
layer78_loop:

        vqrdmulh.s32 q2, q5, q0               // .......*..........................
        vldrw.32 q0, [r_ptr] , #96            // ....*.............................
        vmul.s32 q6, q6, q0                   // ...........*......................
        vldrw.32 q7, [r_ptr, #-32]            // .......................*..........
        vmla.s32 q6, q1, modulus              // .............*....................
        vldrw.32 q1, [r_ptr, #-16]            // ........................*.........
        vmul.s32 q5, q5, q0                   // ......*...........................
        vadd.u32 q0, q4, q6                   // ...............*..................
        vmla.s32 q5, q2, modulus              // ........*.........................
        vsub.u32 q4, q4, q6                   // ..............*...................
        vqrdmulh.s32 q6, q4, q1               // ..........................*.......
        vsub.u32 q2, q3, q5                   // .........*........................
        vmul.s32 q4, q4, q7                   // .........................*........
        vldrw.32 q7, [r_ptr, #-48]            // .................*................
        vmla.s32 q4, q6, modulus              // ...........................*......
        vldrw.32 q1, [r_ptr, #-64]            // ................*.................
        vsub.u32 q6, q2, q4                   // ............................*.....
        vstrw.u32 q6, [in, #-16]              // .................................*
        vadd.u32 q2, q2, q4                   // .............................*....
        vstrw.u32 q2, [in, #-32]              // ................................*.
        vadd.u32 q2, q3, q5                   // ..........*.......................
        vld40.32 {q3,q4,q5,q6}, [in]          // e.................................
        vqrdmulh.s32 q7, q0, q7               // ...................*..............
        vld41.32 {q3,q4,q5,q6}, [in]          // .e................................
        vmul.s32 q1, q0, q1                   // ..................*...............
        vldrw.32 q0, [r_ptr, #16]             // .....e............................
        vmla.s32 q1, q7, modulus              // ....................*.............
        vld42.32 {q3,q4,q5,q6}, [in]          // ..e...............................
        vsub.u32 q7, q2, q1                   // .....................*............
        vld43.32 {q3,q4,q5,q6}, [in]!         // ...e..............................
        vadd.u32 q1, q2, q1                   // ......................*...........
        vstrw.u32 q1, [in, #-128]             // ..............................*...
        vqrdmulh.s32 q1, q6, q0               // ............e.....................
        vstrw.u32 q7, [in, #-112]             // ...............................*..
        
        // original source code
        // vld40.32 {data0,data1,data2,data3}, [in]       // e..............................................
        // vld41.32 {data0,data1,data2,data3}, [in]       // ..e............................................
        // vld42.32 {data0,data1,data2,data3}, [in]       // ......e........................................
        // vld43.32 {data0,data1,data2,data3}, [in]!      // ........e......................................
        // vldrw.32 root0, [r_ptr] , #96                  // ..............*................................
        // vldrw.32 root0_tw, [r_ptr, #-80]               // ....e..........................................
        // vmul.s32 tmp, data2, root0                     // ...................*...........................
        // vqrdmulh.s32 data2, data2, root0_tw            // .............*.................................
        // vmla.s32 tmp, data2, modulus                   // .....................*.........................
        // vsub.u32 data2, data0, tmp                     // ........................*......................
        // vadd.u32 data0, data0, tmp                     // .................................*.............
        // vmul.s32 tmp, data3, root0                     // ...............*...............................
        // vqrdmulh.s32 data3, data3, root0_tw            // ...........e...................................
        // vmla.s32 tmp, data3, modulus                   // .................*.............................
        // vsub.u32 data3, data1, tmp                     // ......................*........................
        // vadd.u32 data1, data1, tmp                     // ....................*..........................
        // vldrw.32 root1, [r_ptr, #-64]                  // ............................*..................
        // vldrw.32 root1_tw, [r_ptr, #-48]               // ..........................*....................
        // vmul.s32 tmp, data1, root1                     // .....................................*.........
        // vqrdmulh.s32 data1, data1, root1_tw            // ...................................*...........
        // vmla.s32 tmp, data1, modulus                   // .......................................*.......
        // vsub.u32 data1, data0, tmp                     // .........................................*.....
        // vadd.u32 data0, data0, tmp                     // ...........................................*...
        // vldrw.32 root2, [r_ptr, #-32]                  // ................*..............................
        // vldrw.32 root2_tw, [r_ptr, #-16]               // ..................*............................
        // vmul.s32 tmp, data3, root2                     // .........................*.....................
        // vqrdmulh.s32 data3, data3, root2_tw            // .......................*.......................
        // vmla.s32 tmp, data3, modulus                   // ...........................*...................
        // vsub.u32 data3, data2, tmp                     // .............................*.................
        // vadd.u32 data2, data2, tmp                     // ...............................*...............
        // vstrw.u32 data0, [in, #-64]                    // ............................................*..
        // vstrw.u32 data1, [in, #-48]                    // ..............................................*
        // vstrw.u32 data2, [in, #-32]                    // ................................*..............
        // vstrw.u32 data3, [in, #-16]                    // ..............................*................
        
        le lr, layer78_loop
        vqrdmulh.s32 q7, q5, q0             // *...........................
        vldrw.32 q0, [r_ptr] , #96          // .*..........................
        vmul.s32 q2, q6, q0                 // ..*.........................
        vldrw.32 q6, [r_ptr, #-32]          // ...*........................
        vmla.s32 q2, q1, modulus            // ....*.......................
        vldrw.32 q1, [r_ptr, #-16]          // .....*......................
        vmul.s32 q0, q5, q0                 // ......*.....................
        vsub.u32 q5, q4, q2                 // .........*..................
        vmla.s32 q0, q7, modulus            // ........*...................
        vadd.u32 q2, q4, q2                 // .......*....................
        vqrdmulh.s32 q7, q5, q1             // ..........*.................
        vsub.u32 q4, q3, q0                 // ...........*................
        vmul.s32 q1, q5, q6                 // ............*...............
        vadd.u32 q0, q3, q0                 // ....................*.......
        vmla.s32 q1, q7, modulus            // ..............*.............
        vldrw.32 q7, [r_ptr, #-48]          // .............*..............
        vsub.u32 q6, q4, q1                 // ................*...........
        vstrw.u32 q6, [in, #-16]            // .................*..........
        vqrdmulh.s32 q5, q2, q7             // .....................*......
        vldrw.32 q6, [r_ptr, #-64]          // ...............*............
        vmul.s32 q3, q2, q6                 // ......................*.....
        vadd.u32 q1, q4, q1                 // ..................*.........
        vmla.s32 q3, q5, modulus            // .......................*....
        vstrw.u32 q1, [in, #-32]            // ...................*........
        vsub.u32 q6, q0, q3                 // ........................*...
        vstrw.u32 q6, [in, #-48]            // ...........................*
        vadd.u32 q0, q0, q3                 // .........................*..
        vstrw.u32 q0, [in, #-64]            // ..........................*.
        
        // original source code
        // vqrdmulh.s32 q2, q5, q0          // *...........................
        // vldrw.32 q0, [r_ptr] , #96       // .*..........................
        // vmul.s32 q6, q6, q0              // ..*.........................
        // vldrw.32 q7, [r_ptr, #-32]       // ...*........................
        // vmla.s32 q6, q1, modulus         // ....*.......................
        // vldrw.32 q1, [r_ptr, #-16]       // .....*......................
        // vmul.s32 q5, q5, q0              // ......*.....................
        // vadd.u32 q0, q4, q6              // .........*..................
        // vmla.s32 q5, q2, modulus         // ........*...................
        // vsub.u32 q4, q4, q6              // .......*....................
        // vqrdmulh.s32 q6, q4, q1          // ..........*.................
        // vsub.u32 q2, q3, q5              // ...........*................
        // vmul.s32 q4, q4, q7              // ............*...............
        // vldrw.32 q7, [r_ptr, #-48]       // ...............*............
        // vmla.s32 q4, q6, modulus         // ..............*.............
        // vldrw.32 q1, [r_ptr, #-64]       // ...................*........
        // vsub.u32 q6, q2, q4              // ................*...........
        // vstrw.u32 q6, [in, #-16]         // .................*..........
        // vadd.u32 q2, q2, q4              // .....................*......
        // vstrw.u32 q2, [in, #-32]         // .......................*....
        // vadd.u32 q2, q3, q5              // .............*..............
        // vqrdmulh.s32 q7, q0, q7          // ..................*.........
        // vmul.s32 q1, q0, q1              // ....................*.......
        // vmla.s32 q1, q7, modulus         // ......................*.....
        // vsub.u32 q7, q2, q1              // ........................*...
        // vadd.u32 q1, q2, q1              // ..........................*.
        // vstrw.u32 q1, [in, #-64]         // ...........................*
        // vstrw.u32 q7, [in, #-48]         // .........................*..
        

        add sp, sp, #STACK_SIZE
        align_stack_undo

        // Restore MVE vector registers
        vpop {d8-d15}
        // Restore GPRs
        pop {r4-r11,lr}
        bx lr