How to decrypt the password using radare2 or gdb?

Question

                                                              .--------------------------------------------------.
                                                              | [0x804851b]                                      |
                                                              | (fcn) main 522                                   |
                                                              |   int main (int argc, char **argv, char **envp); |
                                                              | ; var int32_t var_38h @ ebp-0x38                 |
                                                              | ; var int32_t var_34h @ ebp-0x34                 |
                                                              | ; var int32_t var_30h @ ebp-0x30                 |
                                                              | ; var int32_t var_2ch @ ebp-0x2c                 |
                                                              | ; var int32_t var_28h @ ebp-0x28                 |
                                                              | ; var int32_t var_24h @ ebp-0x24                 |
                                                              | ; var int32_t var_ch @ ebp-0xc                   |
                                                              | ; var int32_t var_8h @ ebp-0x8                   |
                                                              | ; arg int32_t arg_4h @ esp+0x4                   |
                                                              | ; DATA XREF from entry0 @ 0x8048437              |
                                                              | lea ecx, [arg_4h]                                |
                                                              | and esp, 0xfffffff0                              |
                                                              | push dword [ecx - 4]                             |
                                                              | push ebp                                         |
                                                              | mov ebp, esp                                     |
                                                              | push ebx                                         |
                                                              | push ecx                                         |
                                                              | sub esp, 0x30                                    |
                                                              | mov eax, dword gs:[0x14]                         |
                                                              | mov dword [var_ch], eax                          |
                                                              | xor eax, eax                                     |
                                                              | mov dword [var_2ch], 0                           |
                                                              | mov dword [var_38h], 0                           |
                                                              | mov dword [var_34h], 1                           |
                                                              | sub esp, 0xc                                     |
                                                              | ; 0x80487d0                                      |
                                                              | ; "Enter the password: "                         |
                                                              | push str.Enter_the_password:                     |
                                                              | ; int printf(const char *format)                 |
                                                              | call sym.imp.printf;[oa]                         |
                                                              | add esp, 0x10                                    |
                                                              | sub esp, 4                                       |
                                                              | lea eax, [var_24h]                               |
                                                              | ; 20                                             |
                                                              | add eax, 0x14                                    |
                                                              | push eax                                         |
                                                              | lea eax, [var_24h]                               |
                                                              | ; 16                                             |
                                                              | add eax, 0x10                                    |
                                                              | push eax                                         |
                                                              | lea eax, [var_24h]                               |
                                                              | ; 12                                             |
                                                              | add eax, 0xc                                     |
                                                              | push eax                                         |
                                                              | lea eax, [var_24h]                               |
                                                              | add eax, 8                                       |
                                                              | push eax                                         |
                                                              | lea eax, [var_24h]                               |
                                                              | add eax, 4                                       |
                                                              | push eax                                         |
                                                              | lea eax, [var_24h]                               |
                                                              | push eax                                         |
                                                              | ; 0x80487e5                                      |
                                                              | ; "%d %d %d %d %d %d"                            |
                                                              | push str.d__d__d__d__d__d                        |
                                                              | ; int scanf(const char *format)                  |
                                                              | call sym.imp.__isoc99_scanf;[ob]                 |
                                                              | add esp, 0x20                                    |
                                                              | sub esp, 0xc                                     |
                                                              | ; 8                                              |
                                                              | push 8                                           |
                                                              | ;  void *malloc(size_t size)                     |
                                                              | call sym.imp.malloc;[oc]                         |
                                                              | add esp, 0x10                                    |
                                                              | mov dword [var_28h], eax                         |
                                                              | mov eax, dword [var_28h]                         |
                                                              | ; [0x16452:4]=-1                                 |
                                                              | mov dword [eax], 0x16452                         |
                                                              | sub esp, 0xc                                     |
                                                              | ; 8                                              |
                                                              | push 8                                           |
                                                              | ;  void *malloc(size_t size)                     |
                                                              | call sym.imp.malloc;[oc]                         |
                                                              | add esp, 0x10                                    |
                                                              | mov edx, eax                                     |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov dword [eax + 4], edx                         |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | ; [0x16456:4]=-1                                 |
                                                              | mov dword [eax], 0x16456                         |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov ebx, dword [eax + 4]                         |
                                                              | sub esp, 0xc                                     |
                                                              | ; 8                                              |
                                                              | push 8                                           |
                                                              | ;  void *malloc(size_t size)                     |
                                                              | call sym.imp.malloc;[oc]                         |
                                                              | add esp, 0x10                                    |
                                                              | mov dword [ebx + 4], eax                         |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | ; [0x1645d:4]=-1                                 |
                                                              | mov dword [eax], 0x1645d                         |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov ebx, dword [eax + 4]                         |
                                                              | sub esp, 0xc                                     |
                                                              | ; 8                                              |
                                                              | push 8                                           |
                                                              | ;  void *malloc(size_t size)                     |
                                                              | call sym.imp.malloc;[oc]                         |
                                                              | add esp, 0x10                                    |
                                                              | mov dword [ebx + 4], eax                         |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | ; [0x1645e:4]=-1                                 |
                                                              | mov dword [eax], 0x1645e                         |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov ebx, dword [eax + 4]                         |
                                                              | sub esp, 0xc                                     |
                                                              | ; 8                                              |
                                                              | push 8                                           |
                                                              | ;  void *malloc(size_t size)                     |
                                                              | call sym.imp.malloc;[oc]                         |
                                                              | add esp, 0x10                                    |
                                                              | mov dword [ebx + 4], eax                         |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | ; [0x16465:4]=-1                                 |
                                                              | mov dword [eax], 0x16465                         |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov ebx, dword [eax + 4]                         |
                                                              | sub esp, 0xc                                     |
                                                              | ; 8                                              |
                                                              | push 8                                           |
                                                              | ;  void *malloc(size_t size)                     |
                                                              | call sym.imp.malloc;[oc]                         |
                                                              | add esp, 0x10                                    |
                                                              | mov dword [ebx + 4], eax                         |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | ; [0xfffe9bf8:4]=-1                              |
                                                              | mov dword [eax], 0xfffe9bf8                      |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov eax, dword [eax + 4]                         |
                                                              | mov dword [eax + 4], 0                           |
                                                              | mov eax, dword [var_28h]                         |
                                                              | mov dword [var_30h], eax                         |
                                                              | jmp 0x80486d7                                    |
                                                              `--------------------------------------------------'
                                                                  v
                                                                  |
                                                                  '---------.
                                     .----------------------------------------.
                                     |                                      | |
                                     |                                .-----------------------------------.
                                     |                                |  0x80486d7 [oi]                   |
                                     |                                | ; CODE XREF from main @ 0x80486a5 |
                                     |                                | cmp dword [var_30h], 0            |
                                     |                                | jne 0x80486a7                     |
                                     |                                `-----------------------------------'
                                     |                                      t f
                                     |                                      | |
                                     |    .---------------------------------' |
                                     |    |                                   '-----------------------------.
                                     |    |                                                                 |
                                     |.----------------------------------------.                        .---------------------------.
                                     ||  0x80486a7 [of]                        |                        |  0x80486dd [oj]           |
                                     || mov eax, dword [var_30h]               |                        | cmp dword [var_34h], 0    |
                                     || mov ebx, dword [eax]                   |                        | je 0x80486f5              |
                                     || mov eax, dword [var_38h]               |                        `---------------------------'
                                     || mov eax, dword [ebp + eax*4 - 0x24]    |                                f t
                                     || sub esp, 0xc                           |                                | |
                                     || push eax                               |                                | |
                                     || call sym.encrypt;[oe]                  |                                | |
                                     || add esp, 0x10                          |                                | |
                                     || cmp ebx, eax                           |                                | |
                                     || je 0x80486ca                           |                                | |
                                     |`----------------------------------------'                                | |
                                     |        f t                                                               | |
                                     |        | |                                                               | |
                                     |        | '-------------------------------.                               | |
                                     |        '--.                              |                               | |
                                     |           |                              |                               | '---------.
                                     |           |                              |           .-------------------'           |
                                     |           |                              |           |                               |
                                     |       .---------------------------.      |       .---------------------------.   .---------------------------.
                                     |       |  0x80486c3 [og]           |      |       |  0x80486e3 [ol]           |   |  0x80486f5 [om]           |
                                     |       | mov dword [var_34h], 0    |      |       | sub esp, 0xc              |   | sub esp, 0xc              |
                                     |       `---------------------------'      |       | ; 0x80487f7               |   | ; 0x8048800               |
                                     |           v                              |       | ; "Correct!"              |   | ; "Incorrect!"            |
                                     |           |                              |       | push str.Correct          |   | push str.Incorrect        |
                                     |           |                              |       | ; int puts(const char *s) |   | ; int puts(const char *s) |
                                     |           |                              |       | call sym.imp.puts;[ok]    |   | call sym.imp.puts;[ok]    |
                                     |           |                              |       | add esp, 0x10             |   | add esp, 0x10             |
                                     |           |                              |       | jmp 0x8048705             |   `---------------------------'
                                     |           |                              |       `---------------------------'       v
                                     |           |                              |           v                               |
                                     |           |                              |           |                               |
                                     |           '----------.                   |           |                               |
                                     |                      |                   |           '-------------.                 |
                                     |                      |                   |                         | .---------------'
                                     |                      | .-----------------'                         | |
                                     |                      | |                                           | |
                                     |                .-----------------------------.               .-----------------------------------.
                                     |                |  0x80486ca [oh]             |               |  0x8048705 [on]                   |
                                     |                | add dword [var_38h], 1      |               | ; CODE XREF from main @ 0x80486f3 |
                                     |                | mov eax, dword [var_30h]    |               | mov eax, 0                        |
                                     |                | mov eax, dword [eax + 4]    |               | mov ecx, dword [var_ch]           |
                                     |                | mov dword [var_30h], eax    |               | xor ecx, dword gs:[0x14]          |
                                     |                `-----------------------------'               | je 0x804871b                      |
                                     |                    v                                         `-----------------------------------'
                                     |                    |                                                 f t
                                     |                    |                                                 | |
                                     `--------------------'                                                 | |
                                                                                                            | '-----------------.
                                                                                      .---------------------'                   |
                                                                                      |                                         |
                                                                                  .------------------------------------.    .-----------------------.
                                                                                  |  0x8048716 [op]                    |    |  0x804871b [oq]       |
                                                                                  | ; void __stack_chk_fail(void)      |    | lea esp, [var_8h]     |
                                                                                  | call sym.imp.__stack_chk_fail;[oo] |    | pop ecx               |
                                                                                  `------------------------------------'    | pop ebx               |
                                                                                                                            | pop ebp               |
                                                                                                                            | lea esp, [ecx - 4]    |
                                                                                                                            | ret                   |

How to decrypt the password from "call sym.encrypt;[oe]". When I use the breakpoints (radare2) the hex values of the registers are encrypted (i need the value of ebx shown in the image).And what is the significance of the malloc function? Is there non primitive data structure used in this program?

Answer 1

First of All post only relevant code avoid posting some thing that requires horizontal scrolling
second do not post images where you can post copy paste-able code

simplifying the code it is basically

printf ("enter your password\n");
int a,...; 
scanf("%d%d%d%%d%%d%d" , &a,...)
ans1 == encrypt(a) ? "correct" : "incorrect"

where encrypt() is simply a few mathematical operations like add, sub,xor , not on the provided input

the first integer is 4 (5 more ints need to be solved)

>>> hex((((~((4 - 0x18)^0x7a69))^0x11e61)^0x37)+0x26)
'0x16452'  << this is there in your ebx and others are there in the code you posted