Trying to reverse encypter to decrytper in C. Can't figure out what I'm missing

Question

Goal: Trying to reverse encrypter to decrypter that successfully decrypts.

This is the pseudo-code of the disassembled encrypting program. There are no function calls, and I've been able to somewhat replicate this code in C.

undefined8 main(void)

{
  int random;
  time_t current_time;
  long in_FS_OFFSET;
  uint time_number;
  uint rand_seven;
  long counter;
  FILE *open_flag;
  size_t size_of_flag_file;
  void *pointer_allocated_memory_block;
  FILE *open_flag_enc;
  long off_40;
  
  off_40 = *(long *)(in_FS_OFFSET + 40);
  open_flag = fopen("flag","rb");
  fseek(open_flag,0,2);
  size_of_flag_file = ftell(open_flag);
  fseek(open_flag,0,0);
  pointer_allocated_memory_block = malloc(size_of_flag_file);
  fread(pointer_allocated_memory_block,size_of_flag_file,1,open_flag);
  fclose(open_flag);
  current_time = time(NULL);
  time_number = (uint)current_time;
  srand(time_number);
  for (counter = 0; counter < (long)size_of_flag_file; counter = counter + 1) {
    random = rand();
    *(byte *)((long)pointer_allocated_memory_block + counter) = *(byte *)((long)pointer_allocated_memory_block + counter) ^ (byte)random;
    rand_seven = rand();
    rand_seven = rand_seven & 7;
    *(byte *)((long)pointer_allocated_memory_block + counter) = *(byte *)((long)pointer_allocated_memory_block + counter) << (sbyte)rand_seven | *(byte *)((long)pointer_allocated_memory_block + counter) >> 8 - (sbyte)rand_seven;
  }
  open_flag_enc = fopen("flag.enc","wb");
  fwrite(&time_number,1,4,open_flag_enc);
  fwrite(pointer_allocated_memory_block,1,size_of_flag_file,open_flag_enc);
  fclose(open_flag_enc);
  if (off_40 != *(long *)(in_FS_OFFSET + 40)) {
                    /* WARNING: Subroutine does not return */
    __stack_chk_fail();
  }
  return 0;
}

This is the code that needs work: I have tried multiple iterations. If I could get some help or an overview of what I could be/am missing?

#include <stdio.h>
#include <stdlib.h>
#include <time.h>

int main() {
    unsigned int time_number;
    size_t counter;
    FILE *open_flag;
    size_t size_of_flag_file;
    void *pointer_allocated_memory_block;
    FILE *open_flag_enc;

    printf("[DEBUG] Starting program...\n");

    // Open the encrypted file
    printf("[DEBUG] Attempting to open encrypted file...\n");
    open_flag = fopen("flag.enc", "rb");
    if (open_flag == NULL) {
        perror("[ERROR] Could not open file");
        return 1;
    }
    printf("[LOG] Successfully opened encrypted file.\n");

    // Read the RNG seed from the first 4 bytes of the file
    fread(&time_number, 1, 4, open_flag);
    printf("[LOG] RNG Seed read: %u\n", time_number);

    // Determine the size of the remaining file
    fseek(open_flag, 0, SEEK_END);
    size_of_flag_file = ftell(open_flag) - 4;
    fseek(open_flag, 4, SEEK_SET);
    printf("[LOG] Size of remaining file: %zu bytes\n", size_of_flag_file);

    // Read the encrypted flag into memory
    pointer_allocated_memory_block = malloc(size_of_flag_file);
    fread(pointer_allocated_memory_block, 1, size_of_flag_file, open_flag);
    fclose(open_flag);
    printf("[LOG] Successfully read encrypted data into memory.\n");

    // Seed RNG with the same seed
    printf("[DEBUG] Seeding RNG...\n");
    srand(time_number);

    // Decrypt the flag
    printf("[DEBUG] Starting decryption...\n");
    unsigned char *buffer = (unsigned char *)pointer_allocated_memory_block;
    for (counter = 0; counter < size_of_flag_file; counter++) {
        int random = rand();
        int rand_seven = rand() & 7;
        
        buffer[counter] = (buffer[counter] << (8 - rand_seven)) | (buffer[counter] >> rand_seven);
        buffer[counter] ^= (unsigned char)random;
    }
    printf("[LOG] Decryption completed.\n");

    // Write the decrypted flag to "flag.txt"
    printf("[DEBUG] Writing decrypted flag to file...\n");
    open_flag_enc = fopen("flag.txt", "wb");
    fwrite(pointer_allocated_memory_block, 1, size_of_flag_file, open_flag_enc);
    fclose(open_flag_enc);
    printf("[LOG] Successfully wrote decrypted flag to 'flag.txt'.\n");

    // Free allocated memory
    printf("[DEBUG] Freeing allocated memory...\n");
    free(pointer_allocated_memory_block);

    printf("[LOG] Program completed successfully.\n");
    return 0;
}

Thank you for your time.

Answer 1

I'd recommend debugging the decryption loop in your code, and the encryption loop in the original binary, side-by-side and comparing the values at each step. It's possible that your binary's rand() might be producing different values from the one in the original binary -- the behavior of rand() may vary by version of the C standard library, or across implementations of the C standard library. If that's the case, you'd want to replicate their implementation, or just try to use the same standard library that they did.