0

I'm trying to understand what client sends to server. I already did most of the work, but one function left. I can't understand what to do with this function and what it does. I've tried to change function boundaries, but then i get undefined variables and no useful information. I can't attach debugger because program instantly crashes (but it doesn't have any actual anti-debugging measures). I know that function accepts four values:

  1. pointer to int[4] array, which is decrypted before.
  2. pointer to char[8] array, which is encrypted by this function.
  3. int - constant 8.
  4. int - constant 32.

I know that function produces different output every time it is executed with same arguments.
Function:

.text:00088840 ; =============== S U B R O U T I N E =======================================
.text:00088840
.text:00088840 ; Attributes: bp-based frame
.text:00088840
.text:00088840 ; void __fastcall encrypt(int *a1, char *a2, unsigned int a3, int a4)
.text:00088840 encrypt                                 ; CODE XREF: sub_94800+278↓p
.text:00088840
.text:00088840 var_30          = -0x30
.text:00088840 var_2C          = -0x2C
.text:00088840 var_28          = -0x28
.text:00088840 var_24          = -0x24
.text:00088840
.text:00088840 ; __unwind {
.text:00088840                 PUSH            {R4-R7,LR}
.text:00088842                 ADD             R7, SP, #0xC
.text:00088844                 PUSH.W          {R8-R11}
.text:00088848                 SUB             SP, SP, #0x14
.text:0008884A                 STR             R0, [SP,#0x30+var_30]
.text:0008884C                 MOVS            R0, #0
.text:0008884E                 CMP.W           R0, R2,LSR#2
.text:00088852                 STR             R1, [SP,#0x30+var_24]
.text:00088854                 BEQ             loc_88922
.text:00088856                 MOVW            R12, #0x79B9
.text:0008885A                 LSRS            R0, R2, #2
.text:0008885C                 MOV.W           R8, #0
.text:00088860                 MOVT            R12, #0x9E37
.text:00088864                 STR             R0, [SP,#0x30+var_2C]
.text:00088866                 STR             R3, [SP,#0x30+var_28]
.text:00088866 ; End of function encrypt

Pseudocode of function:

void __fastcall encrypt(int *a1, char *a2, unsigned int a3, int a4)
{
  int v4; // [sp+38h] [bp+8h]

  if ( !(a3 >> 2) )
    JUMPOUT(0x88922);
  sub_88868(a3 >> 2, (int)a2, a3, a4, v4);
}

Instructions after function and before next guranteed standalone function (force converted):

.text:00088868 ; =============== S U B R O U T I N E =======================================
.text:00088868
.text:00088868
.text:00088868 sub_88868                               ; CODE XREF: sub_8892A-A↓j
.text:00088868                 PUSH.W          {R0-R11,LR}
.text:0008886C                 MOV.W           R0, #0x150000
.text:00088870                 ADD             R0, SP
.text:00088872                 BL              sub_3E594
.text:00088876                 ADD.W           R0, R0, #0x2D0000
.text:0008887A                 MOV             R1, R0
.text:0008887C                 POP.W           {R0-R11,LR}
.text:00088880                 PUSH.W          {R0-R11,LR}
.text:00088884                 MOV             R0, PC
.text:00088886                 BL              sub_83284
.text:0008888A                 POP.W           {R0-R11,LR}
.text:0008888E                 SUB             SP, SP, #8
.text:00088890                 ADD             SP, SP, #8
.text:00088892                 MOV             R0, R0
.text:00088894                 MOV             R0, R0
.text:00088896                 MOV             R0, R0
.text:00088898                 POP             {PC}
.text:00088898 ; End of function sub_88868
.text:00088898
.text:0008889A ; ---------------------------------------------------------------------------
.text:0008889A                 LDM             R4, {R2,R4,R6,R7}
.text:0008889C                 STRB            R2, [R5,#0x14]
.text:0008889C ; ---------------------------------------------------------------------------
.text:0008889E                 DCB 0x84
.text:0008889F                 DCB 0xEF
.text:000888A0 ; ---------------------------------------------------------------------------
.text:000888A0                 LDR.W           R2, [R11,#0x9D]!
.text:000888A4                 STRB            R4, [R1,R2]
.text:000888A6                 ASRS            R3, R3, #1
.text:000888A8                 BKPT            0x29 ; ')'
.text:000888AA                 BMI             loc_887BA
.text:000888AC                 ADDS            R7, #0x68 ; 'h'
.text:000888AE                 SXTB            R2, R4
.text:000888B0                 LDR             R3, [SP,#0xF8]
.text:000888B2                 STRB            R4, [R6,#5]
.text:000888B4                 BCC             loc_888D0
.text:000888B6                 STR             R6, [R3,#0x18]
.text:000888B8                 MOV             R0, R0
.text:000888BA                 MOV             R0, R0
.text:000888BC                 MOV             R0, R0
.text:000888BE                 LDR             R0, [SP,#0xC]
.text:000888C0                 CMP             R3, #0
.text:000888C2                 LDR.W           R4, [R0,R8,LSL#2]
.text:000888C6                 ADD.W           R0, R0, R8,LSL#2
.text:000888CA                 LDR             R5, [R0,#4]
.text:000888CC                 STR             R0, [SP,#0x10]
.text:000888CE                 BEQ             loc_88910
.text:000888D0
.text:000888D0 loc_888D0                               ; CODE XREF: .text:000888B4↑j
.text:000888D0                 LDR             R0, [SP]
.text:000888D2 ; START OF FUNCTION CHUNK FOR sub_8892A
.text:000888D2
.text:000888D2 loc_888D2                               ; CODE XREF: sub_8892A+6A↓j
.text:000888D2                                         ; sub_8892A+6E↓j
.text:000888D2                 MOV             R11, R12
.text:000888D4                 LDR.W           R9, [R0]
.text:000888D8                 LDR.W           R10, [R0,#4]
.text:000888DC                 LDR             R6, [R0,#8]
.text:000888DE                 LDR             R2, [R0,#0xC]
.text:000888E0
.text:000888E0 loc_888E0                               ; CODE XREF: sub_8892A-1C↓j
.text:000888E0                 ADD.W           LR, R9, R5,LSL#4
.text:000888E4                 ADD.W           R1, R10, R5,LSR#5
.text:000888E8                 EOR.W           R1, R1, LR
.text:000888EC                 ADD.W           R0, R5, R11
.text:000888F0                 EORS            R0, R1
.text:000888F2                 SUBS            R3, #1
.text:000888F4                 ADD             R4, R0
.text:000888F6                 ADD.W           R0, R11, R4
.text:000888FA                 ADD             R11, R12
.text:000888FC                 ADD.W           R1, R6, R4,LSL#4
.text:00088900                 EOR.W           R0, R0, R1
.text:00088904                 ADD.W           R1, R2, R4,LSR#5
.text:00088908                 EOR.W           R0, R0, R1
.text:0008890C                 ADD             R5, R0
.text:0008890E                 BNE             loc_888E0
.text:00088910
.text:00088910 loc_88910                               ; CODE XREF: .text:000888CE↑j
.text:00088910                 LDR             R0, [SP,#-8+arg_10]
.text:00088912                 ADD.W           R8, R8, #2
.text:00088916                 STRD.W          R4, R5, [R0]
.text:0008891A                 LDR             R0, [SP,#-8+arg_4]
.text:0008891C                 LDR             R3, [SP,#-8+arg_8]
.text:0008891E                 CMP             R8, R0
.text:00088920                 BCC             sub_88868
.text:00088922
.text:00088922 loc_88922                               ; CODE XREF: encrypt+14↑j
.text:00088922                 ADD             SP, SP, #0x14
.text:00088924                 POP.W           {R8-R11}
.text:00088928                 POP             {R4-R7,PC}
.text:00088928 ; } // starts at 88840
.text:00088928 ; END OF FUNCTION CHUNK FOR sub_8892A
.text:0008892A
.text:0008892A ; =============== S U B R O U T I N E =======================================
.text:0008892A
.text:0008892A ; Attributes: bp-based frame
.text:0008892A
.text:0008892A sub_8892A                               ; CODE XREF: .text:00036E2E↑p
.text:0008892A                                         ; sub_40134+156↑p
.text:0008892A
.text:0008892A var_34          = -0x34
.text:0008892A var_30          = -0x30
.text:0008892A var_2C          = -0x2C
.text:0008892A var_28          = -0x28
.text:0008892A var_24          = -0x24
.text:0008892A var_20          = -0x20
.text:0008892A var_1C          = -0x1C
.text:0008892A arg_4           =  0xC
.text:0008892A arg_8           =  0x10
.text:0008892A arg_E           =  0x16
.text:0008892A arg_10          =  0x18
.text:0008892A arg_350         =  0x358
.text:0008892A
.text:0008892A ; FUNCTION CHUNK AT .text:000888D2 SIZE 00000058 BYTES
.text:0008892A
.text:0008892A ; __unwind {
.text:0008892A                 PUSH            {R4-R7,LR}
.text:0008892C                 ADD             R7, SP, #0xC
.text:0008892E                 PUSH.W          {R8-R11}
.text:00088932                 SUB             SP, SP, #0x18
.text:00088934                 STR             R0, [SP,#0x34+var_30]
.text:00088936                 MOVS            R0, #0
.text:00088938                 CMP.W           R0, R2,LSR#2
.text:0008893C                 STR             R1, [SP,#0x34+var_24]
.text:0008893E                 BEQ             loc_88A20
.text:00088940                 LSRS            R0, R2, #2
.text:00088942                 MOV             R2, #0x9E3779B9
.text:0008894A                 STR             R0, [SP,#0x34+var_2C]
.text:0008894C                 MUL.W           R0, R3, R2
.text:00088950                 MOVW            R10, #0x8647
.text:00088954                 MOV.W           R9, #0
.text:00088958                 MOVT            R10, #0x61C8
.text:0008895C                 STR             R3, [SP,#0x34+var_28]
.text:0008895E                 STR             R0, [SP,#0x34+var_34]
.text:00088960
.text:00088960 loc_88960                               ; CODE XREF: sub_8892A+F4↓j
.text:00088960                 PUSH.W          {R0-R11,LR}
.text:00088964                 MOV.W           R0, #0x150000
.text:00088968                 ADD             R0, SP
.text:0008896A                 BL              sub_3E594
.text:0008896E                 ADD.W           R0, R0, #0x2D0000
.text:00088972                 MOV             R1, R0
.text:00088974                 POP.W           {R0-R11,LR}
.text:00088978                 PUSH.W          {R0-R11,LR}
.text:0008897C                 MOV             R0, PC
.text:0008897E                 BL              sub_833C8
.text:00088982                 POP.W           {R0-R11,LR}
.text:00088986                 SUB             SP, SP, #8
.text:00088988                 ADD             SP, SP, #8
.text:0008898A                 MOV             R0, R0
.text:0008898C                 MOV             R0, R0
.text:0008898E                 MOV             R0, R0
.text:00088990                 POP             {PC}
.text:00088992 ; ---------------------------------------------------------------------------
.text:00088992                 ADD             R2, SP, #0x30+arg_350
.text:00088994                 BLT             loc_888D2
.text:00088996                 STRB            R1, [R7,#arg_E]
.text:00088998                 BGT             loc_888D2
.text:0008899A                 STRH            R1, [R1,R4]
.text:0008899A ; ---------------------------------------------------------------------------
.text:0008899C                 DCB 0xDD
.text:0008899D                 DCB 0xB7
.text:0008899E ; ---------------------------------------------------------------------------
.text:0008899E                 LSRS            R1, R4, #0x1B
.text:000889A0                 LSLS            R3, R2, #7
.text:000889A2                 ASRS            R4, R6, #0x16
.text:000889A4                 LDR             R4, [R5,#0x48]
.text:000889A6                 CBZ             R4, loc_889F8+2
.text:000889A6 ; ---------------------------------------------------------------------------
.text:000889A8                 DCB    2
.text:000889A9                 DCB 0xF9
.text:000889AA ; ---------------------------------------------------------------------------
.text:000889AA                 LDRH            R4, [R0,#0x2C]
.text:000889AC                 STM             R4, {R0,R1,R3-R5,R7}
.text:000889AE                 STM             R4!, {R0,R1,R5,R6}
.text:000889B0                 MOV             R0, R0
.text:000889B2                 MOV             R0, R0
.text:000889B4                 MOV             R0, R0
.text:000889B6                 LDR             R0, [SP,#0x30+var_20]
.text:000889B8                 CMP             R3, #0
.text:000889BA                 LDR.W           R6, [R0,R9,LSL#2]
.text:000889BE                 ADD.W           R0, R0, R9,LSL#2
.text:000889C2                 LDR             R5, [R0,#4]
.text:000889C4                 STR             R0, [SP,#0x30+var_1C]
.text:000889C6                 BEQ             loc_88A0E
.text:000889C8                 LDR             R0, [SP,#0x30+var_2C]
.text:000889CA                 LDR.W           LR, [SP,#0x30+var_30]
.text:000889CE                 LDR.W           R11, [R0]
.text:000889D2                 LDR             R4, [R0,#4]
.text:000889D4                 LDR             R2, [R0,#8]
.text:000889D6                 LDR.W           R12, [R0,#0xC]
.text:000889DA
.text:000889DA loc_889DA                               ; CODE XREF: sub_8892A+E2↓j
.text:000889DA                 ADD.W           R8, R2, R6,LSL#4
.text:000889DE                 ADD.W           R1, R12, R6,LSR#5
.text:000889E2                 EOR.W           R1, R1, R8
.text:000889E6                 ADD.W           R0, R6, LR
.text:000889EA                 EORS            R0, R1
.text:000889EC                 SUBS            R3, #1
.text:000889EE                 SUB.W           R5, R5, R0
.text:000889F2                 ADD.W           R0, LR, R5
.text:000889F6                 ADD             LR, R10
.text:000889F8
.text:000889F8 loc_889F8                               ; CODE XREF: sub_8892A+7C↑j
.text:000889F8                 ADD.W           R1, R11, R5,LSL#4
.text:000889FC                 EOR.W           R0, R0, R1
.text:00088A00                 ADD.W           R1, R4, R5,LSR#5
.text:00088A04                 EOR.W           R0, R0, R1
.text:00088A08                 SUB.W           R6, R6, R0
.text:00088A0C                 BNE             loc_889DA
.text:00088A0E
.text:00088A0E loc_88A0E                               ; CODE XREF: sub_8892A+9C↑j
.text:00088A0E                 LDR             R0, [SP,#0x30+var_1C]
.text:00088A10                 ADD.W           R9, R9, #2
.text:00088A14                 STRD.W          R6, R5, [R0]
.text:00088A18                 LDR             R0, [SP,#0x30+var_28]
.text:00088A1A                 LDR             R3, [SP,#0x30+var_24]
.text:00088A1C                 CMP             R9, R0
.text:00088A1E                 BCC             loc_88960
.text:00088A20
.text:00088A20 loc_88A20                               ; CODE XREF: sub_8892A+14↑j
.text:00088A20                 ADD             SP, SP, #0x18
.text:00088A22                 POP.W           {R8-R11}
.text:00088A26                 POP             {R4-R7,PC}
.text:00088A26 ; } // starts at 8892A
.text:00088A26 ; End of function sub_8892A

Function call:

.text:00094A6C                 ADD.W           R0, R6, #0x398 ; a1 - int[4] ptr
.text:00094A70                 ADD.W           R1, R6, #0x3B8 ; a2 - char[8] ptr
.text:00094A74                 MOVS            R2, #8  ; a3
.text:00094A76                 MOVS            R3, #0x20 ; ' ' ; a4
.text:00094A78                 BL              encrypt ; addr = 0x88840

(SP is stored in R6)

1

The thing that is unclear to me is what happens in the code block from 00088868 to 00088898. In a plausible scenario, the flow should continue from 000888BE. In that case, this is a TEA encryption function, as can be seen from the 0x9E3779B9 signature, characteristic of the TEA cipher and its modifications like XTEA, and the code at 000888E0, which is equivalent to

R4 += ((R5 << 4) + R9) ^ (R5 + R11) ^ ((R5 >> 5) + R10);
R5 += ((R4 << 4) + R6) ^ (R4 + R11) ^ ((R4 >> 5) + R2);
R11 += R12;

which corresponds to the reference code for TEA encryption:

v0 += ((v1<<4) + k0) ^ (v1 + sum) ^ ((v1>>5) + k1);
v1 += ((v0<<4) + k2) ^ (v0 + sum) ^ ((v0>>5) + k3);
sum += delta;

a1 points to the encryption key, a2 points to the plain text to encrypt, a4 is the number of iterations and a3 seems to be used to pass the plain text length, but I think there is an inconsistency in that it gets stored in [SP,#4] and compared to at 0008891E as a3/4 as if counting 32-bit words, but at 000888C2 the seeming dword counter gets further divided by 4, which may result in multiple encryption passes, although only for a3 greater than 8.

At 0008892A, there is a similar structured decryption function.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.