0

Have an application that only crashes if CPU has SHA extensions. After some investigation find a hashing routine that only occurs if CPU has SHA extensions.

I'm trying to confirm if issue is with input data or a bug in the application, original source code is not available. If a possible bug in the application trying to think what a possible original C code might look like that would generate this type of crash.

The routine has the following disassembly:

X86_64_SHAEXT_SHA1Transform proc near   ; DATA XREF: X86_64_EnableCPUFeatures+14E↑o

arg_0           = qword ptr  8
arg_8           = qword ptr  10h

                mov     [rsp+arg_0], rdi
                mov     [rsp+arg_8], rsi
                mov     rdi, rcx
                mov     rsi, rdx
                mov     rdx, r8
                mov     rax, rsp
                lea     rsp, [rsp-48h]
                movaps  xmmword ptr [rax-48h], xmm6
                movaps  xmmword ptr [rax-38h], xmm7
                movaps  xmmword ptr [rax-28h], xmm8
                movaps  xmmword ptr [rax-18h], xmm9
                movdqu  xmm0, xmmword ptr [rdi]
                movd    xmm1, dword ptr [rdi+10h]
                movdqa  xmm3, cs:xmmword_7FFE8439F4A0
                movdqu  xmm4, xmmword ptr [rsi]
                pshufd  xmm0, xmm0, 1Bh
                movdqu  xmm5, xmmword ptr [rsi+10h]
                pshufd  xmm1, xmm1, 1Bh
                movdqu  xmm6, xmmword ptr [rsi+20h]
                pshufb  xmm4, xmm3
                movdqu  xmm7, xmmword ptr [rsi+30h]
                pshufb  xmm5, xmm3
                pshufb  xmm6, xmm3
                movdqa  xmm9, xmm1
                pshufb  xmm7, xmm3
                jmp     loc_7FFE8439F940
; ---------------------------------------------------------------------------
                align 20h

loc_7FFE8439F940:                       ; CODE XREF: X86_64_SHAEXT_SHA1Transform+74↑j
                                        ; X86_64_SHAEXT_SHA1Transform+28A↓j
                dec     rdx
                lea     rax, [rsi+40h]
                paddd   xmm1, xmm4
                cmovnz  rsi, rax
                movdqa  xmm8, xmm0
                sha1msg1 xmm4, xmm5
                movdqa  xmm2, xmm0
                sha1rnds4 xmm0, xmm1, 0
                sha1nexte xmm2, xmm5
                pxor    xmm4, xmm6
                sha1msg1 xmm5, xmm6
                sha1msg2 xmm4, xmm7
                movdqa  xmm1, xmm0
                sha1rnds4 xmm0, xmm2, 0
                sha1nexte xmm1, xmm6
                pxor    xmm5, xmm7
                sha1msg2 xmm5, xmm4
                sha1msg1 xmm6, xmm7
                movdqa  xmm2, xmm0
                sha1rnds4 xmm0, xmm1, 0
                sha1nexte xmm2, xmm7
                pxor    xmm6, xmm4
                sha1msg1 xmm7, xmm4
                sha1msg2 xmm6, xmm5
                movdqa  xmm1, xmm0
                sha1rnds4 xmm0, xmm2, 0
                sha1nexte xmm1, xmm4
                pxor    xmm7, xmm5
                sha1msg2 xmm7, xmm6
                sha1msg1 xmm4, xmm5
                movdqa  xmm2, xmm0
                sha1rnds4 xmm0, xmm1, 0
                sha1nexte xmm2, xmm5
                pxor    xmm4, xmm6
                sha1msg1 xmm5, xmm6
                sha1msg2 xmm4, xmm7
                movdqa  xmm1, xmm0
                sha1rnds4 xmm0, xmm2, 1
                sha1nexte xmm1, xmm6
                pxor    xmm5, xmm7
                sha1msg2 xmm5, xmm4
                sha1msg1 xmm6, xmm7
                movdqa  xmm2, xmm0
                sha1rnds4 xmm0, xmm1, 1
                sha1nexte xmm2, xmm7
                pxor    xmm6, xmm4
                sha1msg1 xmm7, xmm4
                sha1msg2 xmm6, xmm5
                movdqa  xmm1, xmm0
                sha1rnds4 xmm0, xmm2, 1
                sha1nexte xmm1, xmm4
                pxor    xmm7, xmm5
                sha1msg2 xmm7, xmm6
                sha1msg1 xmm4, xmm5
                movdqa  xmm2, xmm0
                sha1rnds4 xmm0, xmm1, 1
                sha1nexte xmm2, xmm5
                pxor    xmm4, xmm6
                sha1msg1 xmm5, xmm6
                sha1msg2 xmm4, xmm7
                movdqa  xmm1, xmm0
                sha1rnds4 xmm0, xmm2, 1
                sha1nexte xmm1, xmm6
                pxor    xmm5, xmm7
                sha1msg2 xmm5, xmm4
                sha1msg1 xmm6, xmm7
                movdqa  xmm2, xmm0
                sha1rnds4 xmm0, xmm1, 2
                sha1nexte xmm2, xmm7
                pxor    xmm6, xmm4
                sha1msg1 xmm7, xmm4
                sha1msg2 xmm6, xmm5
                movdqa  xmm1, xmm0
                sha1rnds4 xmm0, xmm2, 2
                sha1nexte xmm1, xmm4
                pxor    xmm7, xmm5
                sha1msg2 xmm7, xmm6
                sha1msg1 xmm4, xmm5
                movdqa  xmm2, xmm0
                sha1rnds4 xmm0, xmm1, 2
                sha1nexte xmm2, xmm5
                pxor    xmm4, xmm6
                sha1msg1 xmm5, xmm6
                sha1msg2 xmm4, xmm7
                movdqa  xmm1, xmm0
                sha1rnds4 xmm0, xmm2, 2
                sha1nexte xmm1, xmm6
                pxor    xmm5, xmm7
                sha1msg2 xmm5, xmm4
                sha1msg1 xmm6, xmm7
                movdqa  xmm2, xmm0
                sha1rnds4 xmm0, xmm1, 2
                sha1nexte xmm2, xmm7
                pxor    xmm6, xmm4
                sha1msg1 xmm7, xmm4
                sha1msg2 xmm6, xmm5
                movdqa  xmm1, xmm0
                sha1rnds4 xmm0, xmm2, 3
                sha1nexte xmm1, xmm4
                pxor    xmm7, xmm5
                sha1msg2 xmm7, xmm6
                movdqu  xmm4, xmmword ptr [rsi]
                movdqa  xmm2, xmm0
                sha1rnds4 xmm0, xmm1, 3
                sha1nexte xmm2, xmm5
                movdqu  xmm5, xmmword ptr [rsi+10h]
                pshufb  xmm4, xmm3
                movdqa  xmm1, xmm0
                sha1rnds4 xmm0, xmm2, 3
                sha1nexte xmm1, xmm6
                movdqu  xmm6, xmmword ptr [rsi+20h]
                pshufb  xmm5, xmm3
                movdqa  xmm2, xmm0
                sha1rnds4 xmm0, xmm1, 3
                sha1nexte xmm2, xmm7
                movdqu  xmm7, xmmword ptr [rsi+30h]
                pshufb  xmm6, xmm3
                movdqa  xmm1, xmm0
                sha1rnds4 xmm0, xmm2, 3
                sha1nexte xmm1, xmm9
                pshufb  xmm7, xmm3
                paddd   xmm0, xmm8
                movdqa  xmm9, xmm1
                jnz     loc_7FFE8439F940
                pshufd  xmm0, xmm0, 1Bh
                pshufd  xmm1, xmm1, 1Bh
                movdqu  xmmword ptr [rdi], xmm0
                movd    dword ptr [rdi+10h], xmm1
                movaps  xmm6, xmmword ptr [rax-48h]
                movaps  xmm7, xmmword ptr [rax-38h]
                movaps  xmm8, xmmword ptr [rax-28h]
                movaps  xmm9, xmmword ptr [rax-18h]
                mov     rsp, rax
                mov     rdi, [rsp+arg_0]
                mov     rsi, [rsp+arg_8]
                retn
X86_64_SHAEXT_SHA1Transform endp

Function is called like this:

    00007ffe`8437223d ff15ade82100    call    qword ptr [crypto!g_pSHA1Transform (00007ffe`84590af0)] ds:00007ffe`84590af0={crypto!X86_64_SHAEXT_SHA1Transform (00007ffe`8439f8c0)}
00007ffe`84372243 4803ee          add     rbp,rsi

When function enters, correct return address is on stack:

00007ffe`8439f8c0 48897c2408      mov     qword ptr [rsp+8],rdi ss:00000000`023fb6d0=00008bca8073ce32
0:000> dd @rsp
00000000`023fb6c8  84372243 00007ffe 8073ce32 00008bca <- return address on stack
00000000`023fb6d8  47474747 00000000 00000004 00000000
00000000`023fb6e8  00000060 00000000 00000005 00000000
00000000`023fb6f8  8435e9af 00007ffe 00000078 00000000
00000000`023fb708  00000000 00000000 023fb800 00000000
00000000`023fb718  00000060 00000000 00000000 00000000
00000000`023fb728  00000000 00000000 00000000 00000000
00000000`023fb738  00000000 00000000 00000000 00000000

Here return address is trashed:

00007ffe`8439f8c0 48897c2408       mov     qword ptr [rsp+8], rdi
00007ffe`8439f8c5 4889742410       mov     qword ptr [rsp+10h], rsi
00007ffe`8439f8ca 4889cf           mov     rdi, rcx
00007ffe`8439f8cd 4889d6           mov     rsi, rdx
00007ffe`8439f8d0 4c89c2           mov     rdx, r8
00007ffe`8439f8d3 4889e0           mov     rax, rsp  
00007ffe`8439f8d6 488d6424b8       lea     rsp, [rsp-48h] <- Trashes return address on stack

AFter this executes, return address gone :

0:000> dd @rsp
00000000`023fb680  47474747 47474747 00000000 47474747

When retn executes it's still broken, 0x548ec86043f333c9 doesn't point to any valid memory.

0:000> dd @rsp
00000000`023fb980  43f333c9 548ec860 240b3e9b 8cfc35f9 <- Not a valid memory location on stack

It seems the intention is rsp is backed up in rax, and restored at end of function:

mov     rax, rsp
lea     rsp, [rsp-48h]
; <processing code>
mov     rsp, rax
mov     rdi, qword ptr [rsp+8]
mov     rsi, qword ptr [rsp+10h]
ret     

However rax is changed during function:

lea     rax, [rsi+40h]

If rax hadn't changed the return address would still be correct. Is there a scenario when this code could have actually worked?

  • does rsp get its original value after mov rsp, rax near the end? – Igor Skochinsky Sep 15 at 14:27
  • i'm checking it instruction by instruction, i can see now that must be the intention to restore it – chentiangemalc Sep 15 at 15:11
1

The code is buggy. rax is used to store the original value of rsp and restore it at exit, but it’s also used as a temporary register in the loop in the middle of the function.

I suspect that this function was written manually in assembly but not actually tested on real hardware so the bug went unnoticed or, possibly, modified after initial testing and someone missed the fact that rax is used for two different purposes.

| improve this answer | |

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.