I have a program that read some proprietary files that are compressed. I managed to debug the program and extract the algorithm that decompress the data and use it on my software without any issue. Now I need to implement the compression algorithm but I am wondering if it would be worth the work to reverse the following ASM code as it could be a well known compression algorithm but I am not sure. I already made some tests with zlib an lz77 but it seems that is not the algorithm.

Decompression algorithm:

ConvertShellcode 3.0
Copyright (C) 2009-2015 Alain Rioux (le-tools.com). All rights reserved.

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.

Assembly language source code :
00000000  sub esp,0x18
00000003  push ebp
00000004  push edi
00000005  mov edi,dword[ss:esp+0x24]
00000009  xor eax,eax
0000000b  xor ecx,ecx
0000000d  mov edx,0xc
00000012  mov ah,byte[edi+0xa]
00000015  mov cl,byte[edi+0x8]
00000018  mov al,byte[edi+0x9]
0000001b  shl eax,0x8
0000001e  or eax,ecx
00000020  mov dword[ss:esp+0x1c],eax
00000024  mov eax,0x0
00000029  mov byte[ss:esp+0xb],al
0000002d  mov byte[ss:esp+0xa],al
00000031  mov ebp,eax
00000033  jbe 0x1ab
00000039  push ebx
0000003a  mov ebx,dword[ss:esp+0x2c]
0000003e  push esi
0000003f  mov al,byte[ss:esp+0x12]
00000043  test al,al
00000045  jne 0x54
00000047  mov al,byte[edx+edi*1]
0000004a  mov byte[ss:esp+0x12],0x8
0000004f  mov byte[ss:esp+0x13],al
00000053  inc edx
00000054  test byte[ss:esp+0x13],0x1
00000059  jne 0x68
0000005b  mov cl,byte[edx+edi*1]
0000005e  inc edx
0000005f  mov byte[ebx+ebp*1],cl
00000062  inc ebp
00000063  jmp 0x181
00000068  mov al,byte[edx+edi*1]
0000006b  mov cl,byte[edx+edi*1+0x1]
0000006f  mov byte[ss:esp+0x18],al
00000073  add edx,0x2
00000076  and al,0xf
00000078  mov byte[ss:esp+0x14],cl
0000007c  mov byte[ss:esp+0x20],al
00000080  jne 0xbc
00000082  mov esi,dword[ss:esp+0x14]
00000086  mov eax,dword[ss:esp+0x18]
0000008a  and esi,0xff
00000090  and eax,0xff
00000095  shl esi,0x4
00000098  shr eax,0x4
0000009b  add esi,eax
0000009d  xor eax,eax
0000009f  mov al,byte[edx+edi*1]
000000a2  add eax,0x10
000000a5  je 0xb6
000000a7  mov ecx,ebp
000000a9  sub ecx,esi
000000ab  inc ebp
000000ac  dec eax
000000ad  mov cl,byte[ecx+ebx*1]
000000b0  mov byte[ebx+ebp*1+0xffffffff],cl
000000b4  jne 0xa7
000000b6  inc edx
000000b7  jmp 0x181
000000bc  cmp al,0x1
000000be  jne 0x108
000000c0  mov eax,dword[ss:esp+0x14]
000000c4  mov ecx,dword[ss:esp+0x18]
000000c8  and eax,0xff
000000cd  and ecx,0xff
000000d3  lea edi,[ebx+ebp*1]
000000d6  mov bl,al
000000d8  shr ecx,0x4
000000db  mov bh,bl
000000dd  add ecx,0x3
000000e0  mov eax,ebx
000000e2  mov dword[ss:esp+0x1c],ecx
000000e6  mov esi,ecx
000000e8  shl eax,0x10
000000eb  mov ax,bx
000000ee  mov ebx,dword[ss:esp+0x30]
000000f2  shr ecx,0x2
000000f5  rep stos dword[es:edi]
000000f7  mov ecx,esi
000000f9  and ecx,0x3
000000fc  rep stos byte[es:edi]
000000fe  mov edi,dword[ss:esp+0x2c]
00000102  mov eax,esi
00000104  add ebp,eax
00000106  jmp 0x181
00000108  cmp al,0x2
0000010a  jne 0x14b
0000010c  mov eax,dword[ss:esp+0x14]
00000110  mov ecx,dword[ss:esp+0x18]
00000114  and eax,0xff
00000119  and ecx,0xff
0000011f  shl eax,0x4
00000122  shr ecx,0x4
00000125  lea esi,[edx+edi*1]
00000128  lea edi,[ebx+ebp*1]
0000012b  lea ecx,[eax+ecx*1+0x12]
0000012f  mov dword[ss:esp+0x1c],ecx
00000133  mov eax,ecx
00000135  shr ecx,0x2
00000138  rep movs dword[es:edi],dword[esi]
0000013a  mov ecx,eax
0000013c  add edx,eax
0000013e  and ecx,0x3
00000141  add ebp,eax
00000143  rep movs byte[es:edi],byte[esi]
00000145  mov edi,dword[ss:esp+0x2c]
00000149  jmp 0x181
0000014b  mov esi,dword[ss:esp+0x14]
0000014f  mov ecx,dword[ss:esp+0x18]
00000153  mov eax,dword[ss:esp+0x20]
00000157  and esi,0xff
0000015d  and ecx,0xff
00000163  shl esi,0x4
00000166  shr ecx,0x4
00000169  add esi,ecx
0000016b  and eax,0xff
00000170  jbe 0x181
00000172  mov ecx,ebp
00000174  sub ecx,esi
00000176  inc ebp
00000177  dec eax
00000178  mov cl,byte[ecx+ebx*1]
0000017b  mov byte[ebx+ebp*1+0xffffffff],cl
0000017f  jne 0x172
00000181  mov al,byte[ss:esp+0x12]
00000185  mov cl,byte[ss:esp+0x13]
00000189  dec al
0000018b  mov byte[ss:esp+0x12],al
0000018f  mov eax,dword[ss:esp+0x24]
00000193  shr cl,0x1
00000195  cmp ebp,eax
00000197  mov byte[ss:esp+0x13],cl
0000019b  jb 0x3f
000001a1  pop esi
000001a2  pop ebx
000001a3  mov eax,ebp
000001a5  pop edi
000001a6  pop ebp
000001a7  add esp,0x18
000001aa  ret
000001ab  mov eax,ebp
000001ad  pop edi
000001ae  pop ebp
000001af  add esp,0x18
000001b2  ret

And this is the output of signsrch.exe:

offset   num  description [bits.endian.size]
  00013888 641  CRC-32-IEEE 802.3 [crc32.0x04c11db7 le rev int_min.1024]
  00013888 648  CRC-32-IEEE 802.3 [crc32.0xedb88320 lenorev 1.1024]
  00013c8c 2294 zinflate_lengthExtraBits [32.le.116]
  00013cfd 2304 zinflate_distanceExtraBits [32.be.120]
  00013d00 2303 zinflate_distanceExtraBits [32.le.120]
  000142d0 1086 Zlib dist_code [..512]
  000144d0 1087 Zlib length_code [..256]
  000145d0 1089 Zlib base_length [32.le.116]
  00014644 1091 Zlib base_dist [32.le.120]
  00014738 2291 zinflate_lengthStarts [32.le.116]
  000147b1 2295 zinflate_lengthExtraBits [32.be.116]
  00014830 2298 zinflate_distanceStarts [32.le.120]
  00016dd0 871  ACSS reverse sbox [..256]
  00016dd0 1963 FFT and FHT routines rv_tbl [..128]
  00018ea8 3038 unlzx table_three [32.le.64]

Does anyone knows if this is a known compression algorithm? If no, is there any advice on how to reverse it quickly?

  • 1
    It could've been nice if the addresses in the signsrch and disassembly outputs were aligned...
    – NirIzr
    Oct 26, 2016 at 21:51
  • I'd suggest to try also lzo, lzx and lzw - they are different and as far as I know not compatible.
    – w s
    Oct 27, 2016 at 7:42
  • 1
    You could have saved some time by looking at what this code does. It does not use a dictionary, so it's not an Huffmann type of compression. It strongly smells like an optimized run length encoding, but it could also be something like the original LZ.
    – Jongware
    Oct 27, 2016 at 17:42

1 Answer 1


QuickBMS has a script (comtype_scan) which tries to bruteforce multiple decompression algorthms.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.