From 50afcdca7e1a641094b7427141b3dad2c8d71e8b Mon Sep 17 00:00:00 2001 From: kaleb-himes Date: Tue, 25 Aug 2026 16:03:59 -0600 Subject: [PATCH 1/2] MASM: non-AVX VEX fix, x64 home space, vzeroupper on VAES exits --- wolfcrypt/src/aes_x86_64_asm.asm | 8 + wolfcrypt/src/aes_xts_asm.asm | 8 + wolfcrypt/src/fe_x25519_asm.asm | 520 +++++++++++++++++++++++++++++++ wolfcrypt/src/sha256_asm.asm | 4 + wolfcrypt/src/sha512_asm.asm | 6 +- wolfcrypt/src/sp_x86_64_asm.asm | 10 +- 6 files changed, 549 insertions(+), 7 deletions(-) diff --git a/wolfcrypt/src/aes_x86_64_asm.asm b/wolfcrypt/src/aes_x86_64_asm.asm index 8f5ba53b61b..e5bc739ffd3 100644 --- a/wolfcrypt/src/aes_x86_64_asm.asm +++ b/wolfcrypt/src/aes_x86_64_asm.asm @@ -2373,6 +2373,7 @@ L_AES_ECB_encrypt_vaes_16_aes_enc_block_last: cmp r10d, r11d jb L_AES_ECB_encrypt_vaes_enc_16 L_AES_ECB_encrypt_vaes_done_enc: + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+8] vmovdqu xmm7, OWORD PTR [rsp+24] add rsp, 40 @@ -2657,6 +2658,7 @@ L_AES_ECB_decrypt_vaes_16_aes_dec_block_last: cmp r10d, r11d jb L_AES_ECB_decrypt_vaes_dec_16 L_AES_ECB_decrypt_vaes_done_dec: + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+8] vmovdqu xmm7, OWORD PTR [rsp+24] add rsp, 40 @@ -3034,6 +3036,7 @@ L_AES_CBC_decrypt_vaes_16_aes_dec_block_last: jb L_AES_CBC_decrypt_vaes_dec_16 L_AES_CBC_decrypt_vaes_done_dec: vmovdqu OWORD PTR [r8], xmm8 + vzeroupper vmovdqu xmm6, OWORD PTR [rsp] vmovdqu xmm7, OWORD PTR [rsp+16] vmovdqu xmm8, OWORD PTR [rsp+32] @@ -3504,6 +3507,7 @@ L_AES_CTR_encrypt_vaes_16_aes_enc_block_last: L_AES_CTR_encrypt_vaes_done_enc: vpshufb xmm0, xmm7, xmm8 vmovdqu OWORD PTR [r10], xmm0 + vzeroupper vmovdqu xmm6, OWORD PTR [rsp] vmovdqu xmm7, OWORD PTR [rsp+16] vmovdqu xmm8, OWORD PTR [rsp+32] @@ -3828,6 +3832,7 @@ L_AES_ECB_encrypt_avx512_16_aes_enc_block_last: cmp r10d, r11d jb L_AES_ECB_encrypt_avx512_enc_16 L_AES_ECB_encrypt_avx512_done_enc: + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+8] vmovdqu xmm7, OWORD PTR [rsp+24] vmovdqu xmm8, OWORD PTR [rsp+40] @@ -4149,6 +4154,7 @@ L_AES_ECB_decrypt_avx512_16_aes_dec_block_last: cmp r10d, r11d jb L_AES_ECB_decrypt_avx512_dec_16 L_AES_ECB_decrypt_avx512_done_dec: + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+8] vmovdqu xmm7, OWORD PTR [rsp+24] vmovdqu xmm8, OWORD PTR [rsp+40] @@ -4562,6 +4568,7 @@ L_AES_CBC_decrypt_avx512_16_aes_dec_block_last: jb L_AES_CBC_decrypt_avx512_dec_16 L_AES_CBC_decrypt_avx512_done_dec: vmovdqu OWORD PTR [r8], xmm8 + vzeroupper vmovdqu xmm6, OWORD PTR [rsp] vmovdqu xmm7, OWORD PTR [rsp+16] vmovdqu xmm8, OWORD PTR [rsp+32] @@ -5027,6 +5034,7 @@ L_AES_CTR_encrypt_avx512_16_aes_enc_block_last: L_AES_CTR_encrypt_avx512_done_enc: vpshufb xmm0, xmm7, xmm8 vmovdqu OWORD PTR [r10], xmm0 + vzeroupper vmovdqu xmm6, OWORD PTR [rsp] vmovdqu xmm7, OWORD PTR [rsp+16] vmovdqu xmm8, OWORD PTR [rsp+32] diff --git a/wolfcrypt/src/aes_xts_asm.asm b/wolfcrypt/src/aes_xts_asm.asm index 9804b734d9e..5fe19188a76 100644 --- a/wolfcrypt/src/aes_xts_asm.asm +++ b/wolfcrypt/src/aes_xts_asm.asm @@ -3392,6 +3392,7 @@ L_AES_XTS_encrypt_vaes_last_15_aes_enc_block_last: lea rcx, QWORD PTR [rsi+r13] vmovdqu OWORD PTR [rcx], xmm0 L_AES_XTS_encrypt_vaes_done_enc: + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+72] vmovdqu xmm7, OWORD PTR [rsp+88] vmovdqu xmm8, OWORD PTR [rsp+104] @@ -3872,6 +3873,7 @@ L_AES_XTS_encrypt_update_vaes_last_15_aes_enc_block_last: vmovdqu OWORD PTR [rcx], xmm0 L_AES_XTS_encrypt_update_vaes_done_enc: vmovdqu OWORD PTR [r8], xmm8 + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+64] vmovdqu xmm7, OWORD PTR [rsp+80] vmovdqu xmm8, OWORD PTR [rsp+96] @@ -4461,6 +4463,7 @@ L_AES_XTS_decrypt_vaes_last_31_2_aes_dec_block_last: lea rcx, QWORD PTR [rsi+r13] vmovdqu OWORD PTR [rcx], xmm0 L_AES_XTS_decrypt_vaes_done_dec: + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+72] vmovdqu xmm7, OWORD PTR [rsp+88] vmovdqu xmm8, OWORD PTR [rsp+104] @@ -5015,6 +5018,7 @@ L_AES_XTS_decrypt_update_vaes_last_31_2_aes_dec_block_last: vmovdqu OWORD PTR [rcx], xmm0 L_AES_XTS_decrypt_update_vaes_done_dec: vmovdqu OWORD PTR [r8], xmm8 + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+64] vmovdqu xmm7, OWORD PTR [rsp+80] vmovdqu xmm8, OWORD PTR [rsp+96] @@ -5607,6 +5611,7 @@ L_AES_XTS_encrypt_avx512_last_15_aes_enc_block_last: lea rcx, QWORD PTR [rsi+r13] vmovdqu OWORD PTR [rcx], xmm0 L_AES_XTS_encrypt_avx512_done_enc: + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+72] vmovdqu xmm7, OWORD PTR [rsp+88] vmovdqu xmm8, OWORD PTR [rsp+104] @@ -6094,6 +6099,7 @@ L_AES_XTS_encrypt_update_avx512_last_15_aes_enc_block_last: vmovdqu OWORD PTR [rcx], xmm0 L_AES_XTS_encrypt_update_avx512_done_enc: vmovdqu OWORD PTR [r8], xmm8 + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+64] vmovdqu xmm7, OWORD PTR [rsp+80] vmovdqu xmm8, OWORD PTR [rsp+96] @@ -6700,6 +6706,7 @@ L_AES_XTS_decrypt_avx512_last_31_2_aes_dec_block_last: lea rcx, QWORD PTR [rsi+r13] vmovdqu OWORD PTR [rcx], xmm0 L_AES_XTS_decrypt_avx512_done_dec: + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+72] vmovdqu xmm7, OWORD PTR [rsp+88] vmovdqu xmm8, OWORD PTR [rsp+104] @@ -7271,6 +7278,7 @@ L_AES_XTS_decrypt_update_avx512_last_31_2_aes_dec_block_last: vmovdqu OWORD PTR [rcx], xmm0 L_AES_XTS_decrypt_update_avx512_done_dec: vmovdqu OWORD PTR [r8], xmm8 + vzeroupper vmovdqu xmm6, OWORD PTR [rsp+64] vmovdqu xmm7, OWORD PTR [rsp+80] vmovdqu xmm8, OWORD PTR [rsp+96] diff --git a/wolfcrypt/src/fe_x25519_asm.asm b/wolfcrypt/src/fe_x25519_asm.asm index cbbb99a44e5..3bd50cd61c3 100644 --- a/wolfcrypt/src/fe_x25519_asm.asm +++ b/wolfcrypt/src/fe_x25519_asm.asm @@ -1290,116 +1290,178 @@ fe_invert_x64 PROC mov QWORD PTR [rsp+136], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+136] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+136] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rdx, QWORD PTR [rsp+136] mov rcx, QWORD PTR [rsp+128] add rsp, 152 @@ -3314,116 +3376,178 @@ L_curve25519_base_x64_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov r8, QWORD PTR [rsp+160] ; Multiply ; A[0] * B[0] @@ -5609,116 +5733,178 @@ L_curve25519_x64_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov r9, QWORD PTR [rsp+168] ; Multiply ; A[0] * B[0] @@ -5899,115 +6085,177 @@ fe_pow22523_x64 PROC mov QWORD PTR [rsp+104], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+104] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+104] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, QWORD PTR [rsp+96] mov rdx, rsp mov r8, QWORD PTR [rsp+104] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rdx, QWORD PTR [rsp+104] mov rcx, QWORD PTR [rsp+96] add rsp, 120 @@ -11543,116 +11791,178 @@ fe_invert_avx2 PROC mov QWORD PTR [rsp+136], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+136] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+136] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rdx, QWORD PTR [rsp+136] mov rcx, QWORD PTR [rsp+128] add rsp, 152 @@ -13206,116 +13516,178 @@ L_curve25519_base_avx2_last_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov r8, QWORD PTR [rsp+160] mov rax, QWORD PTR [r8] ; Multiply @@ -15095,116 +15467,178 @@ L_curve25519_avx2_last_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov r9, QWORD PTR [rsp+168] mov rax, QWORD PTR [r9] ; Multiply @@ -15356,115 +15790,177 @@ fe_pow22523_avx2 PROC mov QWORD PTR [rsp+104], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+104] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+104] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, QWORD PTR [rsp+96] mov rdx, rsp mov r8, QWORD PTR [rsp+104] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rdx, QWORD PTR [rsp+104] mov rcx, QWORD PTR [rsp+96] add rsp, 120 @@ -20577,16 +21073,22 @@ L_curve25519_base_avx512_ifma_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+736] vmovdqu xmm7, OWORD PTR [rsp+752] @@ -21395,16 +21897,22 @@ L_curve25519_avx512_ifma_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+744] vmovdqu xmm7, OWORD PTR [rsp+760] @@ -22147,16 +22655,22 @@ L_curve25519_base_avx512_ifma_dq_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+736] vmovdqu xmm7, OWORD PTR [rsp+752] @@ -22920,16 +23434,22 @@ L_curve25519_avx512_ifma_dq_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+744] vmovdqu xmm7, OWORD PTR [rsp+760] diff --git a/wolfcrypt/src/sha256_asm.asm b/wolfcrypt/src/sha256_asm.asm index 4fa488d79e8..be9750ab763 100644 --- a/wolfcrypt/src/sha256_asm.asm +++ b/wolfcrypt/src/sha256_asm.asm @@ -12952,7 +12952,9 @@ Transform_Sha256_AVX2_Len PROC vmovdqu ymm1, YMMWORD PTR [rsi+32] vmovups YMMWORD PTR [rdi+32], ymm0 vmovups YMMWORD PTR [rdi+64], ymm1 + sub rsp, 32 call Transform_Sha256_AVX2 + add rsp, 32 add rsi, 64 sub DWORD PTR [rsp+512], 64 jz L_sha256_len_avx2_done @@ -19521,7 +19523,9 @@ Transform_Sha256_AVX2_RORX_Len PROC vmovdqu ymm1, YMMWORD PTR [rsi+32] vmovups YMMWORD PTR [rdi+32], ymm0 vmovups YMMWORD PTR [rdi+64], ymm1 + sub rsp, 32 call Transform_Sha256_AVX2_RORX + add rsp, 32 add rsi, 64 sub DWORD PTR [rsp+512], 64 jz L_sha256_len_avx2_rorx_done diff --git a/wolfcrypt/src/sha512_asm.asm b/wolfcrypt/src/sha512_asm.asm index 0b95a23dae0..b7cdf467fee 100644 --- a/wolfcrypt/src/sha512_asm.asm +++ b/wolfcrypt/src/sha512_asm.asm @@ -6443,7 +6443,9 @@ Transform_Sha512_AVX2_Len PROC vmovups YMMWORD PTR [rdi+96], ymm1 vmovups YMMWORD PTR [rdi+128], ymm2 vmovups YMMWORD PTR [rdi+160], ymm3 + sub rsp, 32 call Transform_Sha512_AVX2 + add rsp, 32 add QWORD PTR [rdi+224], 128 sub ebp, 128 jz L_sha512_len_avx2_done @@ -9241,9 +9243,9 @@ Transform_Sha512_AVX2_RORX_Len PROC vmovups YMMWORD PTR [rdi+96], ymm1 vmovups YMMWORD PTR [rdi+128], ymm2 vmovups YMMWORD PTR [rdi+160], ymm3 - sub rsp, 8 + sub rsp, 40 call Transform_Sha512_AVX2_RORX - add rsp, 8 + add rsp, 40 pop rsi add QWORD PTR [rdi+224], 128 sub esi, 128 diff --git a/wolfcrypt/src/sp_x86_64_asm.asm b/wolfcrypt/src/sp_x86_64_asm.asm index 690beda8957..d83558296b1 100644 --- a/wolfcrypt/src/sp_x86_64_asm.asm +++ b/wolfcrypt/src/sp_x86_64_asm.asm @@ -44080,7 +44080,7 @@ sp_256_get_point_33_4 PROC movdqu OWORD PTR [rsp+136], xmm14 movdqu OWORD PTR [rsp+152], xmm15 mov rax, 1 - vmovd xmm13, r8d + movd xmm13, r8d add rdx, 200 movd xmm15, eax mov rax, 32 @@ -48107,7 +48107,7 @@ sp_384_get_point_33_6 PROC movdqu OWORD PTR [rsp+136], xmm14 movdqu OWORD PTR [rsp+152], xmm15 mov rax, 1 - vmovd xmm13, r8d + movd xmm13, r8d add rdx, 296 movd xmm15, eax mov rax, 32 @@ -48153,7 +48153,7 @@ L_384_get_point_33_6_start_1: movdqu OWORD PTR [rcx+112], xmm4 movdqu OWORD PTR [rcx+128], xmm5 mov rax, 1 - vmovd xmm13, r8d + movd xmm13, r8d sub rdx, 9472 movd xmm15, eax mov rax, 32 @@ -53437,7 +53437,7 @@ sp_521_get_point_33_9 PROC movdqu OWORD PTR [rsp+144], xmm15 mov r14, 1 mov rax, 1 - vmovd xmm13, r8d + movd xmm13, r8d IFNDEF SP_ALIGN_16 add rdx, 440 ELSE @@ -53503,7 +53503,7 @@ ENDIF movdqu OWORD PTR [rcx+160], xmm5 mov r14, 1 mov rax, 1 - vmovd xmm13, r8d + movd xmm13, r8d IFNDEF SP_ALIGN_16 sub rdx, 14080 ELSE From ca806c89718ae7748b734295282fc1e8a4d701ad Mon Sep 17 00:00:00 2001 From: kaleb-himes Date: Wed, 26 Aug 2026 14:16:26 -0600 Subject: [PATCH 2/2] x86_64 asm: vzeroupper on .S lanes, resync SP asm to generator --- wolfcrypt/src/aes_x86_64_asm.S | 8 + wolfcrypt/src/aes_xts_asm.S | 8 + wolfcrypt/src/sp_x86_64_asm.S | 272 +++++++++++++++++++++----------- wolfcrypt/src/sp_x86_64_asm.asm | 262 +++++++++++++++++++----------- 4 files changed, 369 insertions(+), 181 deletions(-) diff --git a/wolfcrypt/src/aes_x86_64_asm.S b/wolfcrypt/src/aes_x86_64_asm.S index 12f44c28c68..650879988fd 100644 --- a/wolfcrypt/src/aes_x86_64_asm.S +++ b/wolfcrypt/src/aes_x86_64_asm.S @@ -2463,6 +2463,7 @@ L_AES_ECB_encrypt_vaes_16_aes_enc_block_last: cmpl %r9d, %eax jb L_AES_ECB_encrypt_vaes_enc_16 L_AES_ECB_encrypt_vaes_done_enc: + vzeroupper repz retq #ifndef __APPLE__ .size AES_ECB_encrypt_vaes,.-AES_ECB_encrypt_vaes @@ -2749,6 +2750,7 @@ L_AES_ECB_decrypt_vaes_16_aes_dec_block_last: cmpl %r9d, %eax jb L_AES_ECB_decrypt_vaes_dec_16 L_AES_ECB_decrypt_vaes_done_dec: + vzeroupper repz retq #ifndef __APPLE__ .size AES_ECB_decrypt_vaes,.-AES_ECB_decrypt_vaes @@ -3128,6 +3130,7 @@ L_AES_CBC_decrypt_vaes_16_aes_dec_block_last: jb L_AES_CBC_decrypt_vaes_dec_16 L_AES_CBC_decrypt_vaes_done_dec: vmovdqu %xmm8, (%rdx) + vzeroupper popq %r12 repz retq #ifndef __APPLE__ @@ -3599,6 +3602,7 @@ L_AES_CTR_encrypt_vaes_16_aes_enc_block_last: L_AES_CTR_encrypt_vaes_done_enc: vpshufb %xmm8, %xmm7, %xmm0 vmovdqu %xmm0, (%r9) + vzeroupper popq %r12 repz retq #ifndef __APPLE__ @@ -3909,6 +3913,7 @@ L_AES_ECB_encrypt_avx512_16_aes_enc_block_last: cmpl %r9d, %eax jb L_AES_ECB_encrypt_avx512_enc_16 L_AES_ECB_encrypt_avx512_done_enc: + vzeroupper repz retq #ifndef __APPLE__ .size AES_ECB_encrypt_avx512,.-AES_ECB_encrypt_avx512 @@ -4216,6 +4221,7 @@ L_AES_ECB_decrypt_avx512_16_aes_dec_block_last: cmpl %r9d, %eax jb L_AES_ECB_decrypt_avx512_dec_16 L_AES_ECB_decrypt_avx512_done_dec: + vzeroupper repz retq #ifndef __APPLE__ .size AES_ECB_decrypt_avx512,.-AES_ECB_decrypt_avx512 @@ -4621,6 +4627,7 @@ L_AES_CBC_decrypt_avx512_16_aes_dec_block_last: jb L_AES_CBC_decrypt_avx512_dec_16 L_AES_CBC_decrypt_avx512_done_dec: vmovdqu %xmm8, (%rdx) + vzeroupper popq %r12 repz retq #ifndef __APPLE__ @@ -5085,6 +5092,7 @@ L_AES_CTR_encrypt_avx512_16_aes_enc_block_last: L_AES_CTR_encrypt_avx512_done_enc: vpshufb %xmm8, %xmm7, %xmm0 vmovdqu %xmm0, (%r9) + vzeroupper popq %r12 repz retq #ifndef __APPLE__ diff --git a/wolfcrypt/src/aes_xts_asm.S b/wolfcrypt/src/aes_xts_asm.S index d2dd97e5135..d4e48740539 100644 --- a/wolfcrypt/src/aes_xts_asm.S +++ b/wolfcrypt/src/aes_xts_asm.S @@ -3401,6 +3401,7 @@ L_AES_XTS_encrypt_vaes_last_15_aes_enc_block_last: leaq (%rsi,%r13,1), %rcx vmovdqu %xmm0, (%rcx) L_AES_XTS_encrypt_vaes_done_enc: + vzeroupper addq $0x48, %rsp popq %r13 popq %r12 @@ -3866,6 +3867,7 @@ L_AES_XTS_encrypt_update_vaes_last_15_aes_enc_block_last: vmovdqu %xmm0, (%rcx) L_AES_XTS_encrypt_update_vaes_done_enc: vmovdqu %xmm8, (%r8) + vzeroupper addq $0x40, %rsp popq %r12 repz retq @@ -4440,6 +4442,7 @@ L_AES_XTS_decrypt_vaes_last_31_2_aes_dec_block_last: leaq (%rsi,%r13,1), %rcx vmovdqu %xmm0, (%rcx) L_AES_XTS_decrypt_vaes_done_dec: + vzeroupper addq $0x48, %rsp popq %r13 popq %r12 @@ -4979,6 +4982,7 @@ L_AES_XTS_decrypt_update_vaes_last_31_2_aes_dec_block_last: vmovdqu %xmm0, (%rcx) L_AES_XTS_decrypt_update_vaes_done_dec: vmovdqu %xmm8, (%r8) + vzeroupper addq $0x40, %rsp popq %r12 repz retq @@ -5597,6 +5601,7 @@ L_AES_XTS_encrypt_avx512_last_15_aes_enc_block_last: leaq (%rsi,%r13,1), %rcx vmovdqu %xmm0, (%rcx) L_AES_XTS_encrypt_avx512_done_enc: + vzeroupper addq $0x48, %rsp popq %r13 popq %r12 @@ -6069,6 +6074,7 @@ L_AES_XTS_encrypt_update_avx512_last_15_aes_enc_block_last: vmovdqu %xmm0, (%rcx) L_AES_XTS_encrypt_update_avx512_done_enc: vmovdqu %xmm8, (%r8) + vzeroupper addq $0x40, %rsp popq %r12 repz retq @@ -6660,6 +6666,7 @@ L_AES_XTS_decrypt_avx512_last_31_2_aes_dec_block_last: leaq (%rsi,%r13,1), %rcx vmovdqu %xmm0, (%rcx) L_AES_XTS_decrypt_avx512_done_dec: + vzeroupper addq $0x48, %rsp popq %r13 popq %r12 @@ -7216,6 +7223,7 @@ L_AES_XTS_decrypt_update_avx512_last_31_2_aes_dec_block_last: vmovdqu %xmm0, (%rcx) L_AES_XTS_decrypt_update_avx512_done_dec: vmovdqu %xmm8, (%r8) + vzeroupper addq $0x40, %rsp popq %r12 repz retq diff --git a/wolfcrypt/src/sp_x86_64_asm.S b/wolfcrypt/src/sp_x86_64_asm.S index 3994867b4e0..df08343c256 100644 --- a/wolfcrypt/src/sp_x86_64_asm.S +++ b/wolfcrypt/src/sp_x86_64_asm.S @@ -3747,22 +3747,38 @@ _sp_2048_mul_avx2_16: cmpq %rdi, %rbp jne L_end_2048_mul_avx2_16 L_start_2048_mul_avx2_16: - vmovdqu (%rbx), %xmm0 - vmovups %xmm0, (%rdi) - vmovdqu 16(%rbx), %xmm0 - vmovups %xmm0, 16(%rdi) - vmovdqu 32(%rbx), %xmm0 - vmovups %xmm0, 32(%rdi) - vmovdqu 48(%rbx), %xmm0 - vmovups %xmm0, 48(%rdi) - vmovdqu 64(%rbx), %xmm0 - vmovups %xmm0, 64(%rdi) - vmovdqu 80(%rbx), %xmm0 - vmovups %xmm0, 80(%rdi) - vmovdqu 96(%rbx), %xmm0 - vmovups %xmm0, 96(%rdi) - vmovdqu 112(%rbx), %xmm0 - vmovups %xmm0, 112(%rdi) + movq (%rbx), %rax + movq %rax, (%rdi) + movq 8(%rbx), %rax + movq %rax, 8(%rdi) + movq 16(%rbx), %rax + movq %rax, 16(%rdi) + movq 24(%rbx), %rax + movq %rax, 24(%rdi) + movq 32(%rbx), %rax + movq %rax, 32(%rdi) + movq 40(%rbx), %rax + movq %rax, 40(%rdi) + movq 48(%rbx), %rax + movq %rax, 48(%rdi) + movq 56(%rbx), %rax + movq %rax, 56(%rdi) + movq 64(%rbx), %rax + movq %rax, 64(%rdi) + movq 72(%rbx), %rax + movq %rax, 72(%rdi) + movq 80(%rbx), %rax + movq %rax, 80(%rdi) + movq 88(%rbx), %rax + movq %rax, 88(%rdi) + movq 96(%rbx), %rax + movq %rax, 96(%rdi) + movq 104(%rbx), %rax + movq %rax, 104(%rdi) + movq 112(%rbx), %rax + movq %rax, 112(%rdi) + movq 120(%rbx), %rax + movq %rax, 120(%rdi) L_end_2048_mul_avx2_16: addq $0x80, %rsp popq %r14 @@ -7631,18 +7647,30 @@ _sp_2048_sqr_avx2_16: subq $0x80, %rdi cmpq %rdi, %rsi jne L_end_2048_sqr_avx2_16 - vmovdqu (%rbp), %xmm0 - vmovups %xmm0, (%rdi) - vmovdqu 16(%rbp), %xmm0 - vmovups %xmm0, 16(%rdi) - vmovdqu 32(%rbp), %xmm0 - vmovups %xmm0, 32(%rdi) - vmovdqu 48(%rbp), %xmm0 - vmovups %xmm0, 48(%rdi) - vmovdqu 64(%rbp), %xmm0 - vmovups %xmm0, 64(%rdi) - vmovdqu 80(%rbp), %xmm0 - vmovups %xmm0, 80(%rdi) + movq (%rbp), %rax + movq %rax, (%rdi) + movq 8(%rbp), %rax + movq %rax, 8(%rdi) + movq 16(%rbp), %rax + movq %rax, 16(%rdi) + movq 24(%rbp), %rax + movq %rax, 24(%rdi) + movq 32(%rbp), %rax + movq %rax, 32(%rdi) + movq 40(%rbp), %rax + movq %rax, 40(%rdi) + movq 48(%rbp), %rax + movq %rax, 48(%rdi) + movq 56(%rbp), %rax + movq %rax, 56(%rdi) + movq 64(%rbp), %rax + movq %rax, 64(%rdi) + movq 72(%rbp), %rax + movq %rax, 72(%rdi) + movq 80(%rbp), %rax + movq %rax, 80(%rdi) + movq 88(%rbp), %rax + movq %rax, 88(%rdi) L_end_2048_sqr_avx2_16: addq $0x88, %rsp popq %rbx @@ -17757,18 +17785,30 @@ _sp_3072_mul_avx2_12: cmpq %rdi, %rbp jne L_end_3072_mul_avx2_12 L_start_3072_mul_avx2_12: - vmovdqu (%rbx), %xmm0 - vmovups %xmm0, (%rdi) - vmovdqu 16(%rbx), %xmm0 - vmovups %xmm0, 16(%rdi) - vmovdqu 32(%rbx), %xmm0 - vmovups %xmm0, 32(%rdi) - vmovdqu 48(%rbx), %xmm0 - vmovups %xmm0, 48(%rdi) - vmovdqu 64(%rbx), %xmm0 - vmovups %xmm0, 64(%rdi) - vmovdqu 80(%rbx), %xmm0 - vmovups %xmm0, 80(%rdi) + movq (%rbx), %rax + movq %rax, (%rdi) + movq 8(%rbx), %rax + movq %rax, 8(%rdi) + movq 16(%rbx), %rax + movq %rax, 16(%rdi) + movq 24(%rbx), %rax + movq %rax, 24(%rdi) + movq 32(%rbx), %rax + movq %rax, 32(%rdi) + movq 40(%rbx), %rax + movq %rax, 40(%rdi) + movq 48(%rbx), %rax + movq %rax, 48(%rdi) + movq 56(%rbx), %rax + movq %rax, 56(%rdi) + movq 64(%rbx), %rax + movq %rax, 64(%rdi) + movq 72(%rbx), %rax + movq %rax, 72(%rdi) + movq 80(%rbx), %rax + movq %rax, 80(%rdi) + movq 88(%rbx), %rax + movq %rax, 88(%rdi) L_end_3072_mul_avx2_12: addq $0x60, %rsp popq %r12 @@ -22793,12 +22833,18 @@ _sp_3072_sqr_avx2_12: subq $0x60, %rdi cmpq %rdi, %rsi jne L_end_3072_sqr_avx2_12 - vmovdqu (%rbp), %xmm0 - vmovups %xmm0, (%rdi) - vmovdqu 16(%rbp), %xmm0 - vmovups %xmm0, 16(%rdi) - vmovdqu 32(%rbp), %xmm0 - vmovups %xmm0, 32(%rdi) + movq (%rbp), %rax + movq %rax, (%rdi) + movq 8(%rbp), %rax + movq %rax, 8(%rdi) + movq 16(%rbp), %rax + movq %rax, 16(%rdi) + movq 24(%rbp), %rax + movq %rax, 24(%rdi) + movq 32(%rbp), %rax + movq %rax, 32(%rdi) + movq 40(%rbp), %rax + movq %rax, 40(%rdi) movq 48(%rbp), %rax movq %rax, 48(%rdi) L_end_3072_sqr_avx2_12: @@ -45182,7 +45228,7 @@ _sp_256_mont_tpl_4: movq $0xffffffff00000001, %r10 adcq 16(%rsi), %rcx adcq 24(%rsi), %r8 - sbbq $0x00, %r11 + sbbq %r11, %r11 movl %r11d, %r9d andq %r11, %r10 subq %r11, %rdx @@ -45361,7 +45407,7 @@ _sp_256_mont_rsb_sub_dbl_4: movq $0xffffffff00000001, %r15 sbbq %r12, %r8 sbbq %r13, %r9 - sbbq $0x00, %rsi + sbbq %rsi, %rsi movl %esi, %r14d andq %rsi, %r15 addq %rsi, %rax @@ -45436,7 +45482,7 @@ WC_ASM_ATT_HIDDEN(_sp_256_get_point_33_4) _sp_256_get_point_33_4: #endif /* __APPLE__ */ movq $0x01, %rax - vmovd %edx, %xmm13 + movd %edx, %xmm13 addq $0xc8, %rsi movd %eax, %xmm15 movq $32, %rax @@ -49839,7 +49885,7 @@ WC_ASM_ATT_HIDDEN(_sp_384_get_point_33_6) _sp_384_get_point_33_6: #endif /* __APPLE__ */ movq $0x01, %rax - vmovd %edx, %xmm13 + movd %edx, %xmm13 addq $0x128, %rsi movd %eax, %xmm15 movq $32, %rax @@ -49885,7 +49931,7 @@ L_384_get_point_33_6_start_1: movdqu %xmm4, 112(%rdi) movdqu %xmm5, 128(%rdi) movq $0x01, %rax - vmovd %edx, %xmm13 + movd %edx, %xmm13 subq $0x2500, %rsi movd %eax, %xmm15 movq $32, %rax @@ -52539,14 +52585,22 @@ _sp_521_mul_avx2_9: cmpq %rdi, %rbp jne L_end_521_mul_avx2_9 L_start_521_mul_avx2_9: - vmovdqu (%rbx), %xmm0 - vmovups %xmm0, (%rdi) - vmovdqu 16(%rbx), %xmm0 - vmovups %xmm0, 16(%rdi) - vmovdqu 32(%rbx), %xmm0 - vmovups %xmm0, 32(%rdi) - vmovdqu 48(%rbx), %xmm0 - vmovups %xmm0, 48(%rdi) + movq (%rbx), %rax + movq %rax, (%rdi) + movq 8(%rbx), %rax + movq %rax, 8(%rdi) + movq 16(%rbx), %rax + movq %rax, 16(%rdi) + movq 24(%rbx), %rax + movq %rax, 24(%rdi) + movq 32(%rbx), %rax + movq %rax, 32(%rdi) + movq 40(%rbx), %rax + movq %rax, 40(%rdi) + movq 48(%rbx), %rax + movq %rax, 48(%rdi) + movq 56(%rbx), %rax + movq %rax, 56(%rdi) movq 64(%rbx), %rax movq %rax, 64(%rdi) L_end_521_mul_avx2_9: @@ -53373,10 +53427,14 @@ _sp_521_sqr_avx2_9: subq $0x48, %rdi cmpq %rdi, %rsi jne L_end_521_sqr_avx2_9 - vmovdqu (%rbp), %xmm0 - vmovups %xmm0, (%rdi) - vmovdqu 16(%rbp), %xmm0 - vmovups %xmm0, 16(%rdi) + movq (%rbp), %rax + movq %rax, (%rdi) + movq 8(%rbp), %rax + movq %rax, 8(%rdi) + movq 16(%rbp), %rax + movq %rax, 16(%rdi) + movq 24(%rbp), %rax + movq %rax, 24(%rdi) L_end_521_sqr_avx2_9: addq $0x48, %rsp popq %rbx @@ -55472,7 +55530,7 @@ _sp_521_get_point_33_9: pushq %r12 movq $0x01, %r12 movq $0x01, %rax - vmovd %edx, %xmm13 + movd %edx, %xmm13 #ifndef SP_ALIGN_16 addq $0x1b8, %rsi #else @@ -55538,7 +55596,7 @@ L_521_get_point_33_9_start_1: movdqu %xmm5, 160(%rdi) movq $0x01, %r12 movq $0x01, %rax - vmovd %edx, %xmm13 + movd %edx, %xmm13 #ifndef SP_ALIGN_16 subq $0x3700, %rsi #else @@ -63051,22 +63109,38 @@ _sp_1024_mul_avx2_16: cmpq %rdi, %rbp jne L_end_1024_mul_avx2_16 L_start_1024_mul_avx2_16: - vmovdqu (%rbx), %xmm0 - vmovups %xmm0, (%rdi) - vmovdqu 16(%rbx), %xmm0 - vmovups %xmm0, 16(%rdi) - vmovdqu 32(%rbx), %xmm0 - vmovups %xmm0, 32(%rdi) - vmovdqu 48(%rbx), %xmm0 - vmovups %xmm0, 48(%rdi) - vmovdqu 64(%rbx), %xmm0 - vmovups %xmm0, 64(%rdi) - vmovdqu 80(%rbx), %xmm0 - vmovups %xmm0, 80(%rdi) - vmovdqu 96(%rbx), %xmm0 - vmovups %xmm0, 96(%rdi) - vmovdqu 112(%rbx), %xmm0 - vmovups %xmm0, 112(%rdi) + movq (%rbx), %rax + movq %rax, (%rdi) + movq 8(%rbx), %rax + movq %rax, 8(%rdi) + movq 16(%rbx), %rax + movq %rax, 16(%rdi) + movq 24(%rbx), %rax + movq %rax, 24(%rdi) + movq 32(%rbx), %rax + movq %rax, 32(%rdi) + movq 40(%rbx), %rax + movq %rax, 40(%rdi) + movq 48(%rbx), %rax + movq %rax, 48(%rdi) + movq 56(%rbx), %rax + movq %rax, 56(%rdi) + movq 64(%rbx), %rax + movq %rax, 64(%rdi) + movq 72(%rbx), %rax + movq %rax, 72(%rdi) + movq 80(%rbx), %rax + movq %rax, 80(%rdi) + movq 88(%rbx), %rax + movq %rax, 88(%rdi) + movq 96(%rbx), %rax + movq %rax, 96(%rdi) + movq 104(%rbx), %rax + movq %rax, 104(%rdi) + movq 112(%rbx), %rax + movq %rax, 112(%rdi) + movq 120(%rbx), %rax + movq %rax, 120(%rdi) L_end_1024_mul_avx2_16: addq $0x80, %rsp popq %r14 @@ -64115,18 +64189,30 @@ _sp_1024_sqr_avx2_16: subq $0x80, %rdi cmpq %rdi, %rsi jne L_end_1024_sqr_avx2_16 - vmovdqu (%rbp), %xmm0 - vmovups %xmm0, (%rdi) - vmovdqu 16(%rbp), %xmm0 - vmovups %xmm0, 16(%rdi) - vmovdqu 32(%rbp), %xmm0 - vmovups %xmm0, 32(%rdi) - vmovdqu 48(%rbp), %xmm0 - vmovups %xmm0, 48(%rdi) - vmovdqu 64(%rbp), %xmm0 - vmovups %xmm0, 64(%rdi) - vmovdqu 80(%rbp), %xmm0 - vmovups %xmm0, 80(%rdi) + movq (%rbp), %rax + movq %rax, (%rdi) + movq 8(%rbp), %rax + movq %rax, 8(%rdi) + movq 16(%rbp), %rax + movq %rax, 16(%rdi) + movq 24(%rbp), %rax + movq %rax, 24(%rdi) + movq 32(%rbp), %rax + movq %rax, 32(%rdi) + movq 40(%rbp), %rax + movq %rax, 40(%rdi) + movq 48(%rbp), %rax + movq %rax, 48(%rdi) + movq 56(%rbp), %rax + movq %rax, 56(%rdi) + movq 64(%rbp), %rax + movq %rax, 64(%rdi) + movq 72(%rbp), %rax + movq %rax, 72(%rdi) + movq 80(%rbp), %rax + movq %rax, 80(%rdi) + movq 88(%rbp), %rax + movq %rax, 88(%rdi) L_end_1024_sqr_avx2_16: addq $0x88, %rsp popq %rbx diff --git a/wolfcrypt/src/sp_x86_64_asm.asm b/wolfcrypt/src/sp_x86_64_asm.asm index d83558296b1..e764bf9ab69 100644 --- a/wolfcrypt/src/sp_x86_64_asm.asm +++ b/wolfcrypt/src/sp_x86_64_asm.asm @@ -3679,22 +3679,38 @@ sp_2048_mul_avx2_16 PROC cmp rbp, r8 jne L_end_2048_mul_avx2_16 L_start_2048_mul_avx2_16: - vmovdqu xmm0, OWORD PTR [rbx] - vmovups OWORD PTR [r8], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+16] - vmovups OWORD PTR [r8+16], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+32] - vmovups OWORD PTR [r8+32], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+48] - vmovups OWORD PTR [r8+48], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+64] - vmovups OWORD PTR [r8+64], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+80] - vmovups OWORD PTR [r8+80], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+96] - vmovups OWORD PTR [r8+96], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+112] - vmovups OWORD PTR [r8+112], xmm0 + mov rax, QWORD PTR [rbx] + mov QWORD PTR [r8], rax + mov rax, QWORD PTR [rbx+8] + mov QWORD PTR [r8+8], rax + mov rax, QWORD PTR [rbx+16] + mov QWORD PTR [r8+16], rax + mov rax, QWORD PTR [rbx+24] + mov QWORD PTR [r8+24], rax + mov rax, QWORD PTR [rbx+32] + mov QWORD PTR [r8+32], rax + mov rax, QWORD PTR [rbx+40] + mov QWORD PTR [r8+40], rax + mov rax, QWORD PTR [rbx+48] + mov QWORD PTR [r8+48], rax + mov rax, QWORD PTR [rbx+56] + mov QWORD PTR [r8+56], rax + mov rax, QWORD PTR [rbx+64] + mov QWORD PTR [r8+64], rax + mov rax, QWORD PTR [rbx+72] + mov QWORD PTR [r8+72], rax + mov rax, QWORD PTR [rbx+80] + mov QWORD PTR [r8+80], rax + mov rax, QWORD PTR [rbx+88] + mov QWORD PTR [r8+88], rax + mov rax, QWORD PTR [rbx+96] + mov QWORD PTR [r8+96], rax + mov rax, QWORD PTR [rbx+104] + mov QWORD PTR [r8+104], rax + mov rax, QWORD PTR [rbx+112] + mov QWORD PTR [r8+112], rax + mov rax, QWORD PTR [rbx+120] + mov QWORD PTR [r8+120], rax L_end_2048_mul_avx2_16: add rsp, 128 pop rdi @@ -7479,18 +7495,30 @@ sp_2048_sqr_avx2_16 PROC sub r8, 128 cmp r9, r8 jne L_end_2048_sqr_avx2_16 - vmovdqu xmm0, OWORD PTR [rbp] - vmovups OWORD PTR [r8], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+16] - vmovups OWORD PTR [r8+16], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+32] - vmovups OWORD PTR [r8+32], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+48] - vmovups OWORD PTR [r8+48], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+64] - vmovups OWORD PTR [r8+64], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+80] - vmovups OWORD PTR [r8+80], xmm0 + mov rax, QWORD PTR [rbp] + mov QWORD PTR [r8], rax + mov rax, QWORD PTR [rbp+8] + mov QWORD PTR [r8+8], rax + mov rax, QWORD PTR [rbp+16] + mov QWORD PTR [r8+16], rax + mov rax, QWORD PTR [rbp+24] + mov QWORD PTR [r8+24], rax + mov rax, QWORD PTR [rbp+32] + mov QWORD PTR [r8+32], rax + mov rax, QWORD PTR [rbp+40] + mov QWORD PTR [r8+40], rax + mov rax, QWORD PTR [rbp+48] + mov QWORD PTR [r8+48], rax + mov rax, QWORD PTR [rbp+56] + mov QWORD PTR [r8+56], rax + mov rax, QWORD PTR [rbp+64] + mov QWORD PTR [r8+64], rax + mov rax, QWORD PTR [rbp+72] + mov QWORD PTR [r8+72], rax + mov rax, QWORD PTR [rbp+80] + mov QWORD PTR [r8+80], rax + mov rax, QWORD PTR [rbp+88] + mov QWORD PTR [r8+88], rax L_end_2048_sqr_avx2_16: add rsp, 136 pop rbx @@ -17277,18 +17305,30 @@ sp_3072_mul_avx2_12 PROC cmp rbp, r8 jne L_end_3072_mul_avx2_12 L_start_3072_mul_avx2_12: - vmovdqu xmm0, OWORD PTR [rbx] - vmovups OWORD PTR [r8], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+16] - vmovups OWORD PTR [r8+16], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+32] - vmovups OWORD PTR [r8+32], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+48] - vmovups OWORD PTR [r8+48], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+64] - vmovups OWORD PTR [r8+64], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+80] - vmovups OWORD PTR [r8+80], xmm0 + mov rax, QWORD PTR [rbx] + mov QWORD PTR [r8], rax + mov rax, QWORD PTR [rbx+8] + mov QWORD PTR [r8+8], rax + mov rax, QWORD PTR [rbx+16] + mov QWORD PTR [r8+16], rax + mov rax, QWORD PTR [rbx+24] + mov QWORD PTR [r8+24], rax + mov rax, QWORD PTR [rbx+32] + mov QWORD PTR [r8+32], rax + mov rax, QWORD PTR [rbx+40] + mov QWORD PTR [r8+40], rax + mov rax, QWORD PTR [rbx+48] + mov QWORD PTR [r8+48], rax + mov rax, QWORD PTR [rbx+56] + mov QWORD PTR [r8+56], rax + mov rax, QWORD PTR [rbx+64] + mov QWORD PTR [r8+64], rax + mov rax, QWORD PTR [rbx+72] + mov QWORD PTR [r8+72], rax + mov rax, QWORD PTR [rbx+80] + mov QWORD PTR [r8+80], rax + mov rax, QWORD PTR [rbx+88] + mov QWORD PTR [r8+88], rax L_end_3072_mul_avx2_12: add rsp, 96 pop r14 @@ -22173,12 +22213,18 @@ sp_3072_sqr_avx2_12 PROC sub r8, 96 cmp r9, r8 jne L_end_3072_sqr_avx2_12 - vmovdqu xmm0, OWORD PTR [rbp] - vmovups OWORD PTR [r8], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+16] - vmovups OWORD PTR [r8+16], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+32] - vmovups OWORD PTR [r8+32], xmm0 + mov rax, QWORD PTR [rbp] + mov QWORD PTR [r8], rax + mov rax, QWORD PTR [rbp+8] + mov QWORD PTR [r8+8], rax + mov rax, QWORD PTR [rbp+16] + mov QWORD PTR [r8+16], rax + mov rax, QWORD PTR [rbp+24] + mov QWORD PTR [r8+24], rax + mov rax, QWORD PTR [rbp+32] + mov QWORD PTR [r8+32], rax + mov rax, QWORD PTR [rbp+40] + mov QWORD PTR [r8+40], rax mov rax, QWORD PTR [rbp+48] mov QWORD PTR [r8+48], rax L_end_3072_sqr_avx2_12: @@ -43853,7 +43899,7 @@ sp_256_mont_tpl_4 PROC mov r12, 18446744069414584321 adc r9, QWORD PTR [rdx+16] adc r10, QWORD PTR [rdx+24] - sbb r13, 0 + sbb r13, r13 mov r11d, r13d and r12, r13 sub rax, r13 @@ -44005,7 +44051,7 @@ sp_256_mont_rsb_sub_dbl_4 PROC mov rsi, 18446744069414584321 sbb r10, r14 sbb r11, r15 - sbb rdx, 0 + sbb rdx, rdx mov edi, edx and rsi, rdx add rax, rdx @@ -50664,14 +50710,22 @@ sp_521_mul_avx2_9 PROC cmp rbp, r8 jne L_end_521_mul_avx2_9 L_start_521_mul_avx2_9: - vmovdqu xmm0, OWORD PTR [rbx] - vmovups OWORD PTR [r8], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+16] - vmovups OWORD PTR [r8+16], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+32] - vmovups OWORD PTR [r8+32], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+48] - vmovups OWORD PTR [r8+48], xmm0 + mov rax, QWORD PTR [rbx] + mov QWORD PTR [r8], rax + mov rax, QWORD PTR [rbx+8] + mov QWORD PTR [r8+8], rax + mov rax, QWORD PTR [rbx+16] + mov QWORD PTR [r8+16], rax + mov rax, QWORD PTR [rbx+24] + mov QWORD PTR [r8+24], rax + mov rax, QWORD PTR [rbx+32] + mov QWORD PTR [r8+32], rax + mov rax, QWORD PTR [rbx+40] + mov QWORD PTR [r8+40], rax + mov rax, QWORD PTR [rbx+48] + mov QWORD PTR [r8+48], rax + mov rax, QWORD PTR [rbx+56] + mov QWORD PTR [r8+56], rax mov rax, QWORD PTR [rbx+64] mov QWORD PTR [r8+64], rax L_end_521_mul_avx2_9: @@ -51483,10 +51537,14 @@ sp_521_sqr_avx2_9 PROC sub r8, 72 cmp r9, r8 jne L_end_521_sqr_avx2_9 - vmovdqu xmm0, OWORD PTR [rbp] - vmovups OWORD PTR [r8], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+16] - vmovups OWORD PTR [r8+16], xmm0 + mov rax, QWORD PTR [rbp] + mov QWORD PTR [r8], rax + mov rax, QWORD PTR [rbp+8] + mov QWORD PTR [r8+8], rax + mov rax, QWORD PTR [rbp+16] + mov QWORD PTR [r8+16], rax + mov rax, QWORD PTR [rbp+24] + mov QWORD PTR [r8+24], rax L_end_521_sqr_avx2_9: add rsp, 72 pop rbx @@ -60838,22 +60896,38 @@ sp_1024_mul_avx2_16 PROC cmp rbp, r8 jne L_end_1024_mul_avx2_16 L_start_1024_mul_avx2_16: - vmovdqu xmm0, OWORD PTR [rbx] - vmovups OWORD PTR [r8], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+16] - vmovups OWORD PTR [r8+16], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+32] - vmovups OWORD PTR [r8+32], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+48] - vmovups OWORD PTR [r8+48], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+64] - vmovups OWORD PTR [r8+64], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+80] - vmovups OWORD PTR [r8+80], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+96] - vmovups OWORD PTR [r8+96], xmm0 - vmovdqu xmm0, OWORD PTR [rbx+112] - vmovups OWORD PTR [r8+112], xmm0 + mov rax, QWORD PTR [rbx] + mov QWORD PTR [r8], rax + mov rax, QWORD PTR [rbx+8] + mov QWORD PTR [r8+8], rax + mov rax, QWORD PTR [rbx+16] + mov QWORD PTR [r8+16], rax + mov rax, QWORD PTR [rbx+24] + mov QWORD PTR [r8+24], rax + mov rax, QWORD PTR [rbx+32] + mov QWORD PTR [r8+32], rax + mov rax, QWORD PTR [rbx+40] + mov QWORD PTR [r8+40], rax + mov rax, QWORD PTR [rbx+48] + mov QWORD PTR [r8+48], rax + mov rax, QWORD PTR [rbx+56] + mov QWORD PTR [r8+56], rax + mov rax, QWORD PTR [rbx+64] + mov QWORD PTR [r8+64], rax + mov rax, QWORD PTR [rbx+72] + mov QWORD PTR [r8+72], rax + mov rax, QWORD PTR [rbx+80] + mov QWORD PTR [r8+80], rax + mov rax, QWORD PTR [rbx+88] + mov QWORD PTR [r8+88], rax + mov rax, QWORD PTR [rbx+96] + mov QWORD PTR [r8+96], rax + mov rax, QWORD PTR [rbx+104] + mov QWORD PTR [r8+104], rax + mov rax, QWORD PTR [rbx+112] + mov QWORD PTR [r8+112], rax + mov rax, QWORD PTR [rbx+120] + mov QWORD PTR [r8+120], rax L_end_1024_mul_avx2_16: add rsp, 128 pop rdi @@ -61895,18 +61969,30 @@ sp_1024_sqr_avx2_16 PROC sub r8, 128 cmp r9, r8 jne L_end_1024_sqr_avx2_16 - vmovdqu xmm0, OWORD PTR [rbp] - vmovups OWORD PTR [r8], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+16] - vmovups OWORD PTR [r8+16], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+32] - vmovups OWORD PTR [r8+32], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+48] - vmovups OWORD PTR [r8+48], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+64] - vmovups OWORD PTR [r8+64], xmm0 - vmovdqu xmm0, OWORD PTR [rbp+80] - vmovups OWORD PTR [r8+80], xmm0 + mov rax, QWORD PTR [rbp] + mov QWORD PTR [r8], rax + mov rax, QWORD PTR [rbp+8] + mov QWORD PTR [r8+8], rax + mov rax, QWORD PTR [rbp+16] + mov QWORD PTR [r8+16], rax + mov rax, QWORD PTR [rbp+24] + mov QWORD PTR [r8+24], rax + mov rax, QWORD PTR [rbp+32] + mov QWORD PTR [r8+32], rax + mov rax, QWORD PTR [rbp+40] + mov QWORD PTR [r8+40], rax + mov rax, QWORD PTR [rbp+48] + mov QWORD PTR [r8+48], rax + mov rax, QWORD PTR [rbp+56] + mov QWORD PTR [r8+56], rax + mov rax, QWORD PTR [rbp+64] + mov QWORD PTR [r8+64], rax + mov rax, QWORD PTR [rbp+72] + mov QWORD PTR [r8+72], rax + mov rax, QWORD PTR [rbp+80] + mov QWORD PTR [r8+80], rax + mov rax, QWORD PTR [rbp+88] + mov QWORD PTR [r8+88], rax L_end_1024_sqr_avx2_16: add rsp, 136 pop rbx