I have a program that read some proprietary files that are compressed. I managed to debug the program and extract the algorithm that decompress the data and use it on my software without any issue. Now I need to implement the compression algorithm but I am wondering if it would be worth the work to reverse the following ASM code as it could be a well known compression algorithm but I am not sure. I already made some tests with zlib an lz77 but it seems that is not the algorithm.
Decompression algorithm:
ConvertShellcode 3.0
***********************************************************************
Copyright (C) 2009-2015 Alain Rioux (le-tools.com). All rights reserved.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***********************************************************************
Assembly language source code :
***************************************
00000000 sub esp,0x18
00000003 push ebp
00000004 push edi
00000005 mov edi,dword[ss:esp+0x24]
00000009 xor eax,eax
0000000b xor ecx,ecx
0000000d mov edx,0xc
00000012 mov ah,byte[edi+0xa]
00000015 mov cl,byte[edi+0x8]
00000018 mov al,byte[edi+0x9]
0000001b shl eax,0x8
0000001e or eax,ecx
00000020 mov dword[ss:esp+0x1c],eax
00000024 mov eax,0x0
00000029 mov byte[ss:esp+0xb],al
0000002d mov byte[ss:esp+0xa],al
00000031 mov ebp,eax
00000033 jbe 0x1ab
00000039 push ebx
0000003a mov ebx,dword[ss:esp+0x2c]
0000003e push esi
0000003f mov al,byte[ss:esp+0x12]
00000043 test al,al
00000045 jne 0x54
00000047 mov al,byte[edx+edi*1]
0000004a mov byte[ss:esp+0x12],0x8
0000004f mov byte[ss:esp+0x13],al
00000053 inc edx
00000054 test byte[ss:esp+0x13],0x1
00000059 jne 0x68
0000005b mov cl,byte[edx+edi*1]
0000005e inc edx
0000005f mov byte[ebx+ebp*1],cl
00000062 inc ebp
00000063 jmp 0x181
00000068 mov al,byte[edx+edi*1]
0000006b mov cl,byte[edx+edi*1+0x1]
0000006f mov byte[ss:esp+0x18],al
00000073 add edx,0x2
00000076 and al,0xf
00000078 mov byte[ss:esp+0x14],cl
0000007c mov byte[ss:esp+0x20],al
00000080 jne 0xbc
00000082 mov esi,dword[ss:esp+0x14]
00000086 mov eax,dword[ss:esp+0x18]
0000008a and esi,0xff
00000090 and eax,0xff
00000095 shl esi,0x4
00000098 shr eax,0x4
0000009b add esi,eax
0000009d xor eax,eax
0000009f mov al,byte[edx+edi*1]
000000a2 add eax,0x10
000000a5 je 0xb6
000000a7 mov ecx,ebp
000000a9 sub ecx,esi
000000ab inc ebp
000000ac dec eax
000000ad mov cl,byte[ecx+ebx*1]
000000b0 mov byte[ebx+ebp*1+0xffffffff],cl
000000b4 jne 0xa7
000000b6 inc edx
000000b7 jmp 0x181
000000bc cmp al,0x1
000000be jne 0x108
000000c0 mov eax,dword[ss:esp+0x14]
000000c4 mov ecx,dword[ss:esp+0x18]
000000c8 and eax,0xff
000000cd and ecx,0xff
000000d3 lea edi,[ebx+ebp*1]
000000d6 mov bl,al
000000d8 shr ecx,0x4
000000db mov bh,bl
000000dd add ecx,0x3
000000e0 mov eax,ebx
000000e2 mov dword[ss:esp+0x1c],ecx
000000e6 mov esi,ecx
000000e8 shl eax,0x10
000000eb mov ax,bx
000000ee mov ebx,dword[ss:esp+0x30]
000000f2 shr ecx,0x2
000000f5 rep stos dword[es:edi]
000000f7 mov ecx,esi
000000f9 and ecx,0x3
000000fc rep stos byte[es:edi]
000000fe mov edi,dword[ss:esp+0x2c]
00000102 mov eax,esi
00000104 add ebp,eax
00000106 jmp 0x181
00000108 cmp al,0x2
0000010a jne 0x14b
0000010c mov eax,dword[ss:esp+0x14]
00000110 mov ecx,dword[ss:esp+0x18]
00000114 and eax,0xff
00000119 and ecx,0xff
0000011f shl eax,0x4
00000122 shr ecx,0x4
00000125 lea esi,[edx+edi*1]
00000128 lea edi,[ebx+ebp*1]
0000012b lea ecx,[eax+ecx*1+0x12]
0000012f mov dword[ss:esp+0x1c],ecx
00000133 mov eax,ecx
00000135 shr ecx,0x2
00000138 rep movs dword[es:edi],dword[esi]
0000013a mov ecx,eax
0000013c add edx,eax
0000013e and ecx,0x3
00000141 add ebp,eax
00000143 rep movs byte[es:edi],byte[esi]
00000145 mov edi,dword[ss:esp+0x2c]
00000149 jmp 0x181
0000014b mov esi,dword[ss:esp+0x14]
0000014f mov ecx,dword[ss:esp+0x18]
00000153 mov eax,dword[ss:esp+0x20]
00000157 and esi,0xff
0000015d and ecx,0xff
00000163 shl esi,0x4
00000166 shr ecx,0x4
00000169 add esi,ecx
0000016b and eax,0xff
00000170 jbe 0x181
00000172 mov ecx,ebp
00000174 sub ecx,esi
00000176 inc ebp
00000177 dec eax
00000178 mov cl,byte[ecx+ebx*1]
0000017b mov byte[ebx+ebp*1+0xffffffff],cl
0000017f jne 0x172
00000181 mov al,byte[ss:esp+0x12]
00000185 mov cl,byte[ss:esp+0x13]
00000189 dec al
0000018b mov byte[ss:esp+0x12],al
0000018f mov eax,dword[ss:esp+0x24]
00000193 shr cl,0x1
00000195 cmp ebp,eax
00000197 mov byte[ss:esp+0x13],cl
0000019b jb 0x3f
000001a1 pop esi
000001a2 pop ebx
000001a3 mov eax,ebp
000001a5 pop edi
000001a6 pop ebp
000001a7 add esp,0x18
000001aa ret
000001ab mov eax,ebp
000001ad pop edi
000001ae pop ebp
000001af add esp,0x18
000001b2 ret
And this is the output of signsrch.exe:
offset num description [bits.endian.size]
--------------------------------------------
00013888 641 CRC-32-IEEE 802.3 [crc32.0x04c11db7 le rev int_min.1024]
00013888 648 CRC-32-IEEE 802.3 [crc32.0xedb88320 lenorev 1.1024]
00013c8c 2294 zinflate_lengthExtraBits [32.le.116]
00013cfd 2304 zinflate_distanceExtraBits [32.be.120]
00013d00 2303 zinflate_distanceExtraBits [32.le.120]
000142d0 1086 Zlib dist_code [..512]
000144d0 1087 Zlib length_code [..256]
000145d0 1089 Zlib base_length [32.le.116]
00014644 1091 Zlib base_dist [32.le.120]
00014738 2291 zinflate_lengthStarts [32.le.116]
000147b1 2295 zinflate_lengthExtraBits [32.be.116]
00014830 2298 zinflate_distanceStarts [32.le.120]
00016dd0 871 ACSS reverse sbox [..256]
00016dd0 1963 FFT and FHT routines rv_tbl [..128]
00018ea8 3038 unlzx table_three [32.le.64]
Does anyone knows if this is a known compression algorithm? If no, is there any advice on how to reverse it quickly?