how to find the encryption algorithm?

Question

I have an arm complied so file and find the decrypt function which pseudocode generated by IDA Pro is like this:

char __cdecl EncodeUtil::getDecryptStr()
{
  int *v0; // r0
  int *v1; // r7
  unsigned int i; // r5
  char v3; // r6
  int v5; // [sp+4h] [bp-1Ch]

  v1 = v0;
  HttpUtility::URLDecode(&v5);
  for ( i = 0; i < *(_DWORD *)(v5 - 12); ++i )
  {
    sub_3B25D0(&v5);
    v3 = byte_41A7DD[i & 7]; //byte_41A7DD     DCB 0xC, 0x17, 0xDE, 0x22, 0x2C, 0xC9, 0x37, 0x43
    *(_BYTE *)(v5 + i) ^= v3;
    sub_3B25D0(&v5);
    if ( !*(_BYTE *)(v5 + i) )
    {
      sub_3B25D0(&v5);
      *(_BYTE *)(v5 + i) ^= v3;
    }
  }
  sub_3B2E20(v1, &v5);
  sub_3B1CCC(&v5);
  return (char)v1;
}

and the sub_3B25D0 function is:

int *__fastcall sub_3B25D0(int *result)
{
  if ( *(_DWORD *)(*result - 4) >= 0 )
    result = sub_3B2580(result);
  return result;
}

and the sub_3B2580 function is:

int *__fastcall sub_3B2580(int *result)
{
  int v1; // r3
  int *v2; // r4

  v1 = *result;
  v2 = result;
  if ( (int *)(*result - 12) != &dword_4C60C0 )
  {
    if ( *(_DWORD *)(v1 - 4) > 0 )
    {
      result = sub_3B1D0C(result, 0, 0, 0);
      v1 = *v2;
    }
    *(_DWORD *)(v1 - 4) = -1;
  }
  return result;
}

and then sub_3B1D0C

int *__fastcall sub_3B1D0C(int *result, size_t a2, int a3, int a4)
{
  int v4; // r12
  int v5; // r10
  int v6; // r8
  int v7; // r7
  unsigned int v8; // r3
  unsigned int v9; // r8
  int *v10; // r5
  int v11; // r4
  size_t v12; // r6
  size_t v13; // r10
  int v14; // r0
  int v15; // r9
  bool v16; // zf
  int v17; // r7
  int v18; // r4
  const void *v19; // r1
  int v20; // r7
  int v21; // r4
  _BYTE *v22; // r1
  char v23; // [sp+4h] [bp-24h]

  v4 = *result;
  v5 = *(_DWORD *)(*result - 12);
  v6 = a4 - a3;
  v7 = a4;
  v8 = *(_DWORD *)(*result - 8);
  v9 = v6 + v5;
  v10 = result;
  v11 = a3;
  v12 = a2;
  v13 = v5 - a2 - a3;
  if ( v9 > v8 || *(_DWORD *)(v4 - 4) > 0 )
  {
    v14 = sub_3B1B30(v9, v8, &v23);
    if ( v12 )
    {
      v22 = (_BYTE *)*v10;
      if ( v12 == 1 )
      {
        *(_BYTE *)(v14 + 12) = *v22;
        v15 = v14 + 12;
      }
      else
      {
        v15 = v14 + 12;
        memcpy((void *)(v14 + 12), v22, v12);
      }
    }
    else
    {
      v15 = v14 + 12;
    }
    if ( v13 )
    {
      v20 = v7 + v12;
      v21 = v11 + v12;
      if ( v13 == 1 )
        *(_BYTE *)(v15 + v20) = *(_BYTE *)(*v10 + v21);
      else
        memcpy((void *)(v15 + v20), (const void *)(*v10 + v21), v13);
    }
    result = (int *)(*v10 - 12);
    if ( result != &dword_4C60C0 )
      result = (int *)sub_3B1C84();
    v4 = v15;
    *v10 = v15;
  }
  else
  {
    v16 = a3 == v7;
    if ( a3 != v7 )
      v16 = v13 == 0;
    if ( !v16 )
    {
      v17 = v7 + a2;
      v18 = a3 + a2;
      result = (int *)(v4 + v17);
      v19 = (const void *)(v4 + a3 + a2);
      if ( v13 == 1 )
        *(_BYTE *)(v4 + v17) = *(_BYTE *)(v4 + v18);
      else
        result = (int *)memmove(result, v19, v13);
      v4 = *v10;
    }
  }
  if ( (int *)(v4 - 12) != &dword_4C60C0 )
  {
    *(_DWORD *)(v4 - 4) = 0;
    *(_DWORD *)(v4 - 12) = v9;
    *(_BYTE *)(v4 + v9) = 0;
  }
  return result;
}

can anyone give a best guess what does the function do after HttpUtility::URLDecode ?

EDIT

As inspired by @NirIzr's answer below and some of the comments, I wrote a snippet of Java code to try the XOR like this:

 public static void main(String args[])
    {
        byte[] _bytes = null;
        byte[] key = {(byte) 0xC,(byte)0x17,(byte)0xDE,(byte)0x22,(byte)0x2C,(byte)0xC9,(byte)0x37,(byte)0x43};
        String s = "%EB%9Ff%C5%A4q%D0%D9%88%F2M%87%C9Z%92%A6%83%BC%3B%86%8B%2D%8B%EC";
        try {
            String decoded = URLDecoder.decode(s,"UTF-8");
            _bytes = decoded.getBytes();
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }


        for(int i=0;i<_bytes.length;i++){
            _bytes[i] = (byte)(_bytes[i] ^ key[i%key.length]);
        }

        System.out.println(new String(_bytes));

    }

But the out put didn't seems to be good:

�cD�mF�����v��c͓tm���1��&���c�v�����͓t��

Answer 1

All the credit for this this answer should go to the @NirIzr and @Avery3R.

Here is a python script that implements the xoring mentioned by these kind people:

import urllib
import binascii
data = "%EB%9Ff%C5%A4q%D0%D9%88%F2M%87%C9Z%92%A6%83%BC%3B%86%8B%2D%8B%EC" #quoted data as is
unquoted = urllib.unquote(data)
key = binascii.unhexlify("0C17DE222CC93743") # the key
idx = 0
res = ""
for c in unquoted:
    res += chr(ord(c) ^ ord(key[idx % len(key)]))
    idx += 1
print res

The result of running:

╭─wireshrink@[cenzored] ~/test  
╰─$ python ./test.py 
爸爸的哥哥叫大伯

I think that you either forgot to encode it back to UTF8 before printing or there is some other issue in your code or default environment that is related to encoding.

Answer 2

First, I have to say several things don't quite add up in the decompilation, and I often find it easier to understand disassembly than decompilation

Let's start with the function doing most of the work - sub_3B25D0.

As this function is somewhat long and has a little bit of quote-unquote scary math in it, our first impression may be that this function does all or most of the encryption heavy lifting. However after further investigation (and i won't dwell into this here) you can clearly see indications of an compression algorithm. There are multiple key elements revealing that, and you can find a full explanation of how to differentiate compression from encryption in this answer.

I'll however go over some of the key indications as they appear in the code snippet you provided.

1. There are no neat iteration of all characters in the string, by order. Nearly all encryption algorithms, either Block or Stream iterate over all characters of the input string. Blocks will iterate over block-sized chunks of the string.
2. Not enough substitutions over either input message or stream/state.
3. Plenty of partial memcpy/memmoves, indicating this function composes as string out of different sub-sequences of input string.

Therefore, lets rename sub_3B1D0C to NI_BlockDecompression. You may use your own initials. I'm assuming it's a decompression and not a compression function merely because you called the flow "Decrypt", impling input is intangible.

Now, going bottom to top, we can briefly review sub_3B2580 and sub_3B25D0 and see those functions just run a couple of simple sanity checks on input but basically don't do anything interesting. after a brief peek it looks like there's some kind of protection against running the NI_BlockDecompression function more than once on the same string, not sure though. Let's just skip them and get to the second interesting function - getDecryptStr.

I find the multiple calls to sub_3B25D0 quite odd, and I suggest you look further into what's going on there. The EncodeUtil::getDecryptStr function just seem to xor bytes with the hardcoded "XOR encryption" key at byte_41A7DD.