Have an application that only crashes if CPU has SHA extensions. After some investigation find a hashing routine that only occurs if CPU has SHA extensions.
I'm trying to confirm if issue is with input data or a bug in the application, original source code is not available. If a possible bug in the application trying to think what a possible original C code might look like that would generate this type of crash.
The routine has the following disassembly:
X86_64_SHAEXT_SHA1Transform proc near ; DATA XREF: X86_64_EnableCPUFeatures+14E↑o
arg_0 = qword ptr 8
arg_8 = qword ptr 10h
mov [rsp+arg_0], rdi
mov [rsp+arg_8], rsi
mov rdi, rcx
mov rsi, rdx
mov rdx, r8
mov rax, rsp
lea rsp, [rsp-48h]
movaps xmmword ptr [rax-48h], xmm6
movaps xmmword ptr [rax-38h], xmm7
movaps xmmword ptr [rax-28h], xmm8
movaps xmmword ptr [rax-18h], xmm9
movdqu xmm0, xmmword ptr [rdi]
movd xmm1, dword ptr [rdi+10h]
movdqa xmm3, cs:xmmword_7FFE8439F4A0
movdqu xmm4, xmmword ptr [rsi]
pshufd xmm0, xmm0, 1Bh
movdqu xmm5, xmmword ptr [rsi+10h]
pshufd xmm1, xmm1, 1Bh
movdqu xmm6, xmmword ptr [rsi+20h]
pshufb xmm4, xmm3
movdqu xmm7, xmmword ptr [rsi+30h]
pshufb xmm5, xmm3
pshufb xmm6, xmm3
movdqa xmm9, xmm1
pshufb xmm7, xmm3
jmp loc_7FFE8439F940
; ---------------------------------------------------------------------------
align 20h
loc_7FFE8439F940: ; CODE XREF: X86_64_SHAEXT_SHA1Transform+74↑j
; X86_64_SHAEXT_SHA1Transform+28A↓j
dec rdx
lea rax, [rsi+40h]
paddd xmm1, xmm4
cmovnz rsi, rax
movdqa xmm8, xmm0
sha1msg1 xmm4, xmm5
movdqa xmm2, xmm0
sha1rnds4 xmm0, xmm1, 0
sha1nexte xmm2, xmm5
pxor xmm4, xmm6
sha1msg1 xmm5, xmm6
sha1msg2 xmm4, xmm7
movdqa xmm1, xmm0
sha1rnds4 xmm0, xmm2, 0
sha1nexte xmm1, xmm6
pxor xmm5, xmm7
sha1msg2 xmm5, xmm4
sha1msg1 xmm6, xmm7
movdqa xmm2, xmm0
sha1rnds4 xmm0, xmm1, 0
sha1nexte xmm2, xmm7
pxor xmm6, xmm4
sha1msg1 xmm7, xmm4
sha1msg2 xmm6, xmm5
movdqa xmm1, xmm0
sha1rnds4 xmm0, xmm2, 0
sha1nexte xmm1, xmm4
pxor xmm7, xmm5
sha1msg2 xmm7, xmm6
sha1msg1 xmm4, xmm5
movdqa xmm2, xmm0
sha1rnds4 xmm0, xmm1, 0
sha1nexte xmm2, xmm5
pxor xmm4, xmm6
sha1msg1 xmm5, xmm6
sha1msg2 xmm4, xmm7
movdqa xmm1, xmm0
sha1rnds4 xmm0, xmm2, 1
sha1nexte xmm1, xmm6
pxor xmm5, xmm7
sha1msg2 xmm5, xmm4
sha1msg1 xmm6, xmm7
movdqa xmm2, xmm0
sha1rnds4 xmm0, xmm1, 1
sha1nexte xmm2, xmm7
pxor xmm6, xmm4
sha1msg1 xmm7, xmm4
sha1msg2 xmm6, xmm5
movdqa xmm1, xmm0
sha1rnds4 xmm0, xmm2, 1
sha1nexte xmm1, xmm4
pxor xmm7, xmm5
sha1msg2 xmm7, xmm6
sha1msg1 xmm4, xmm5
movdqa xmm2, xmm0
sha1rnds4 xmm0, xmm1, 1
sha1nexte xmm2, xmm5
pxor xmm4, xmm6
sha1msg1 xmm5, xmm6
sha1msg2 xmm4, xmm7
movdqa xmm1, xmm0
sha1rnds4 xmm0, xmm2, 1
sha1nexte xmm1, xmm6
pxor xmm5, xmm7
sha1msg2 xmm5, xmm4
sha1msg1 xmm6, xmm7
movdqa xmm2, xmm0
sha1rnds4 xmm0, xmm1, 2
sha1nexte xmm2, xmm7
pxor xmm6, xmm4
sha1msg1 xmm7, xmm4
sha1msg2 xmm6, xmm5
movdqa xmm1, xmm0
sha1rnds4 xmm0, xmm2, 2
sha1nexte xmm1, xmm4
pxor xmm7, xmm5
sha1msg2 xmm7, xmm6
sha1msg1 xmm4, xmm5
movdqa xmm2, xmm0
sha1rnds4 xmm0, xmm1, 2
sha1nexte xmm2, xmm5
pxor xmm4, xmm6
sha1msg1 xmm5, xmm6
sha1msg2 xmm4, xmm7
movdqa xmm1, xmm0
sha1rnds4 xmm0, xmm2, 2
sha1nexte xmm1, xmm6
pxor xmm5, xmm7
sha1msg2 xmm5, xmm4
sha1msg1 xmm6, xmm7
movdqa xmm2, xmm0
sha1rnds4 xmm0, xmm1, 2
sha1nexte xmm2, xmm7
pxor xmm6, xmm4
sha1msg1 xmm7, xmm4
sha1msg2 xmm6, xmm5
movdqa xmm1, xmm0
sha1rnds4 xmm0, xmm2, 3
sha1nexte xmm1, xmm4
pxor xmm7, xmm5
sha1msg2 xmm7, xmm6
movdqu xmm4, xmmword ptr [rsi]
movdqa xmm2, xmm0
sha1rnds4 xmm0, xmm1, 3
sha1nexte xmm2, xmm5
movdqu xmm5, xmmword ptr [rsi+10h]
pshufb xmm4, xmm3
movdqa xmm1, xmm0
sha1rnds4 xmm0, xmm2, 3
sha1nexte xmm1, xmm6
movdqu xmm6, xmmword ptr [rsi+20h]
pshufb xmm5, xmm3
movdqa xmm2, xmm0
sha1rnds4 xmm0, xmm1, 3
sha1nexte xmm2, xmm7
movdqu xmm7, xmmword ptr [rsi+30h]
pshufb xmm6, xmm3
movdqa xmm1, xmm0
sha1rnds4 xmm0, xmm2, 3
sha1nexte xmm1, xmm9
pshufb xmm7, xmm3
paddd xmm0, xmm8
movdqa xmm9, xmm1
jnz loc_7FFE8439F940
pshufd xmm0, xmm0, 1Bh
pshufd xmm1, xmm1, 1Bh
movdqu xmmword ptr [rdi], xmm0
movd dword ptr [rdi+10h], xmm1
movaps xmm6, xmmword ptr [rax-48h]
movaps xmm7, xmmword ptr [rax-38h]
movaps xmm8, xmmword ptr [rax-28h]
movaps xmm9, xmmword ptr [rax-18h]
mov rsp, rax
mov rdi, [rsp+arg_0]
mov rsi, [rsp+arg_8]
retn
X86_64_SHAEXT_SHA1Transform endp
Function is called like this:
00007ffe`8437223d ff15ade82100 call qword ptr [crypto!g_pSHA1Transform (00007ffe`84590af0)] ds:00007ffe`84590af0={crypto!X86_64_SHAEXT_SHA1Transform (00007ffe`8439f8c0)}
00007ffe`84372243 4803ee add rbp,rsi
When function enters, correct return address is on stack:
00007ffe`8439f8c0 48897c2408 mov qword ptr [rsp+8],rdi ss:00000000`023fb6d0=00008bca8073ce32
0:000> dd @rsp
00000000`023fb6c8 84372243 00007ffe 8073ce32 00008bca <- return address on stack
00000000`023fb6d8 47474747 00000000 00000004 00000000
00000000`023fb6e8 00000060 00000000 00000005 00000000
00000000`023fb6f8 8435e9af 00007ffe 00000078 00000000
00000000`023fb708 00000000 00000000 023fb800 00000000
00000000`023fb718 00000060 00000000 00000000 00000000
00000000`023fb728 00000000 00000000 00000000 00000000
00000000`023fb738 00000000 00000000 00000000 00000000
Here return address is trashed:
00007ffe`8439f8c0 48897c2408 mov qword ptr [rsp+8], rdi
00007ffe`8439f8c5 4889742410 mov qword ptr [rsp+10h], rsi
00007ffe`8439f8ca 4889cf mov rdi, rcx
00007ffe`8439f8cd 4889d6 mov rsi, rdx
00007ffe`8439f8d0 4c89c2 mov rdx, r8
00007ffe`8439f8d3 4889e0 mov rax, rsp
00007ffe`8439f8d6 488d6424b8 lea rsp, [rsp-48h] <- Trashes return address on stack
AFter this executes, return address gone :
0:000> dd @rsp
00000000`023fb680 47474747 47474747 00000000 47474747
When retn executes it's still broken, 0x548ec86043f333c9 doesn't point to any valid memory.
0:000> dd @rsp
00000000`023fb980 43f333c9 548ec860 240b3e9b 8cfc35f9 <- Not a valid memory location on stack
It seems the intention is rsp is backed up in rax, and restored at end of function:
mov rax, rsp
lea rsp, [rsp-48h]
; <processing code>
mov rsp, rax
mov rdi, qword ptr [rsp+8]
mov rsi, qword ptr [rsp+10h]
ret
However rax is changed during function:
lea rax, [rsi+40h]
If rax hadn't changed the return address would still be correct. Is there a scenario when this code could have actually worked?
rspget its original value aftermov rsp, raxnear the end?