Reverse Engineering a Pwn Challenge with IDA Pro: Practical Analysis Techniques


In binary analysis, understanding data type representations is fundamental. Here are common data type definitions:

  • _WORD (1 word = 2 bytes = 16 bits)
  • _DWORD (2 words = 4 bytes = 32 bits)
  • _QWORD (4 words = 8 bytes = 64 bits)
  • _OWORD (8 words = 16 bytes = 128 bits)

Assembly directives for data definition:

  • DB (Define Byte): Defines single-byte (8-bit) data
  • DW (Define Word): Defines word-sized data (typically 2 bytes/16 bits)
  • DD (Define Doubleword): Defines doubleword-sized data (typically 4 bytes/32 bits)
  • DQ (Define Quadword): Defines quadword-sized data (typically 8 bytes/64 bits)

Module Interacsions: Import and Export Functions

In modular programming, functions can be shared between different modules through import and export mechanisms:

  • Export functions: Functions made accessible to other modules. In Windows, this is typically done with __declspec(dllexport) in DLLs.
  • Import functions: Functions referenced from external modules. This requires including appropriate headers and linking against the defining module.

Locating the Program Entry Point

To begin reverse engineering, we first identify the program's entry point. In most cases, we look for the start function, which then typically cals the main function.

IDA Configuration and Renaming

Proper configuration of IDA is crucial for effective analysis. Key steps include:

  1. Setting appropriate analysis options
  2. Renaming functions and variables for clarity
  3. Creating custom structures to represent complex data

Main Function Analysis

Let's examine the main function from our target binary:

__int64 __fastcall main(int argc, char **argv, char **envp)
{
  int status; // [rsp+Ch] [rbp-14h]
  void *input_buffer; // [rsp+10h] [rbp-10h]
  void *tokenized_input; // [rsp+18h] [rbp-8h]

  setvbuf(stdin, 0LL, 2, 0LL);
  setvbuf(stdout, 0LL, 2, 0LL);
  setvbuf(stderr, 0LL, 2, 0LL);
  alarm(0x20u);
  global_data = create_memory_block();
  global_data->flag = 0;
  global_data->current_dir = global_data;
  global_data->path[0] = 47;
  strncpy(current_path, global_data->path, 1uLL);
  
  do
  {
    printf("\x1B[36mmybash:%s$ \x1B[0m", current_path);
    input_buffer = read_input();
    tokenized_input = tokenize_input(input_buffer);
    status = process_command(tokenized_input);
    free(input_buffer);
    free(tokenized_input);
  }
  while ( status );
  
  return 0LL;
}

Structure Creation

When analyzing functions that manipulate complex data, creating custom structures can greatly improve readability. Consider this memory allocation function:

MemoryBlock *create_memory_block()
{
  MemoryBlock *block; // [rsp+8h] [rbp-8h]

  block = malloc(0x40uLL);
  if (!block)
    _exit(0);
  memset(&block->reserved[4], 0, 0x10uLL);
  block->reserved[7] = 0LL;
  block->reserved[5] = 0LL;
  block->reserved[4] = 0LL;
  block->reserved[6] = 0LL;
  return block;
}

By defining a custom structure, we can improve the clarity:

MemoryBlock *create_memory_block()
{
  MemoryBlock *block; // [rsp+8h] [rbp-8h]

  block = (MemoryBlock *)malloc(0x40uLL);
  if (!block)
    _exit(0);
  memset(block->reserved_area, 0, sizeof(block->reserved_area));
  block->pointer56 = 0LL;
  block->pointer40 = 0LL;
  block->pointer32 = 0LL;
  block->pointer48 = 0LL;
  return block;
}

Input Handling Functions

Let's examine the input reading function:

char *read_input()
{
  int capacity; // [rsp+Ch] [rbp-14h]
  int position; // [rsp+10h] [rbp-10h]
  int current_char; // [rsp+14h] [rbp-Ch]
  char *input_buffer; // [rsp+18h] [rbp-8h]

  capacity = 528;
  position = 0;
  input_buffer = (char *)malloc(0x210uLL);
  if (!input_buffer)
    _exit(0);
  
  while (1)
  {
    current_char = getchar();
    if (current_char == -1 || current_char == 10)
      break;
    input_buffer[position++] = current_char;
    
    if (position >= capacity && capacity <= 2147483639)
    {
      capacity += 496;
      input_buffer = (char *)realloc(input_buffer, capacity);
      if (!input_buffer)
      {
        fwrite("mybash: allocation error\n", 1uLL, 0x19uLL, stderr);
        exit(1);
      }
    }
  }
  input_buffer[position] = 0;
  return input_buffer;
}

String Tokenization

The tokenization function splits input into command and arguments:

char **tokenize_input(char *input)
{
  int capacity; // [rsp+10h] [rbp-20h]
  int current_pos; // [rsp+14h] [rbp-1Ch]
  char **tokens; // [rsp+18h] [rbp-18h]
  char *current_token; // [rsp+20h] [rbp-10h]
  char **temp_buffer; // [rsp+28h] [rbp-8h]

  capacity = 64;
  current_pos = 0;
  temp_buffer = (char **)malloc(0x200uLL);
  if (!temp_buffer)
    _exit(0);
  tokens = temp_buffer;
  
  for (current_token = strtok(input, " \t\r\n\a"); current_token; 
       current_token = strtok(0LL, " \t\r\n\a"))
  {
    tokens[current_pos++] = current_token;
    if (current_pos >= capacity)
    {
      capacity += 64;
      tokens = (char **)realloc(tokens, 8LL * capacity);
      if (!tokens)
      {
        fwrite("mybash: allocation error\n", 1uLL, 0x19uLL, stderr);
        exit(1);
      }
    }
  }
  tokens[current_pos] = 0LL;
  return tokens;
}

Command Processing

The command processing function dispatches commands to their respective handlers:

__int64 process_command(char **args)
{
  int i; // [rsp+1Ch] [rbp-4h]

  if (!*args)
    return 1LL;
  
  for (i = 0; i < get_command_count(); ++i)
  {
    if (!strcmp(*args, command_names[i]))
      return command_handlers[i](args);
  }
  
  printf("%s: ERROR\n", *args);
  return 0LL;
}

Command Implementations

Let's look at a few command implementations:

touch Command

__int64 command_touch(char **args)
{
  int param_index; // [rsp+1Ch] [rbp-14h]
  MemoryBlock *first_file; // [rsp+20h] [rbp-10h]
  MemoryBlock *new_file; // [rsp+28h] [rbp-8h]

  param_index = 1;
  if (!args[1])
    fwrite("mybash: missing operand\n", 1uLL, 0x18uLL, stderr);
  
  while (args[param_index])
  {
    first_file = current_directory->first_file;
    if (folder_exists(first_file, args[param_index]) != 1)
    {
      ++param_index;
    }
    else
    {
      new_file = create_memory_block();
      new_file->is_file = 1;
      set_filename(new_file, args[param_index]);
      
      if (current_directory->first_file)
        link_files(first_file, new_file);
      else
        current_directory->first_file = new_file;
      
      ++param_index;
    }
  }
  return 1LL;
}

mkdir Command

__int64 command_mkdir(const char **args)
{
  int param_index; // [rsp+1Ch] [rbp-14h]
  MemoryBlock *first_file; // [rsp+20h] [rbp-10h]
  MemoryBlock *new_dir; // [rsp+28h] [rbp-8h]

  param_index = 1;
  if (!args[1])
    fwrite("mybash: missing operand\n", 1uLL, 0x18uLL, stderr);
  
  while (args[param_index])
  {
    first_file = current_directory->first_file;
    if (folder_exists(first_file, args[param_index]) != 1)
    {
      fprintf(stderr, "mybash: cannot create directory '%s'\n", args[param_index++]);
    }
    else
    {
      new_dir = create_memory_block();
      new_dir->is_file = 0;  // 0 indicates directory
      new_dir->parent = current_directory;
      set_filename(new_dir, args[param_index]);
      
      if (current_directory->first_file)
        link_files(first_file, new_dir);
      else
        current_directory->first_file = new_dir;
      
      ++param_index;
    }
  }
  return 1LL;
}

cat Command

__int64 command_cat(const char **args)
{
  unsigned __int64 max_param_length; // rax
  void *stack_buffer; // rsp
  _BYTE v4[8]; // [rsp+8h] [rbp-70h] BYREF
  const char **saved_args; // [rsp+10h] [rbp-68h]
  char v6; // [rsp+1Fh] [rbp-59h]
  int max_param_length_plus1; // [rsp+20h] [rbp-58h]
  int param_index; // [rsp+24h] [rbp-54h]
  MemoryBlock *first_file; // [rsp+28h] [rbp-50h]
  __int64 max_param_length_minus; // [rsp+30h] [rbp-48h]
  char *filename; // [rsp+38h] [rbp-40h]
  unsigned __int64 v12; // [rsp+40h] [rbp-38h]

  saved_args = args;
  v12 = __readfsqword(0x28u);
  param_index = 0;
  v6 = 0;
  max_param_length_plus1 = 0;
  first_file = 0LL;
  
  while (saved_args[++param_index])
  {
    if (strlen(saved_args[param_index]) > max_param_length_plus1)
      max_param_length_plus1 = strlen(saved_args[param_index]) + 1;
  }
  
  max_param_length_minus = max_param_length_plus1 - 1LL;
  max_param_length = 16 * ((max_param_length_plus1 + 15LL) / 0x10uLL);
  while (v4 != &v4[-(max_param_length & 0xFFFFFFFFFFFFF000LL)])
    ;
  stack_buffer = alloca(max_param_length & 0xFFF);
  
  if ((max_param_length & 0xFFF) != 0)
    *&v4[(max_param_length & 0xFFF) - 8] = *&v4[(max_param_length & 0xFFF) - 8];
  filename = v4;
  
  param_index = 1;
  
  if (!saved_args[1])
    fwrite("mybash: missing operand\n", 1uLL, 0x18uLL, stderr);
  
  while (saved_args[param_index])
  {
    first_file = current_directory->first_file;
    strcpy(filename, saved_args[param_index]);
    
    while (first_file)
    {
      if (!strcmp(filename, first_file->filename))
      {
        if (first_file->content)
          puts(first_file->content);
        v6 = 1;
        break;
      }
      first_file = first_file->next_file;
    }
    
    if (v6 != 1)
      fprintf(stderr, "mybash: %s: No such file or directory\n", saved_args[param_index]);
    ++param_index;
  }
  
  return 1LL;
}

rm Command

__int64 command_rm(const char **args)
{
  int param_index; // [rsp+1Ch] [rbp-14h]
  MemoryBlock *file_to_remove; // [rsp+20h] [rbp-10h]
  int success; // [rsp+28h] [rbp-8h]

  param_index = 1;
  if (!args[1])
    fwrite("mybash: missing operand\n", 1uLL, 0x18uLL, stderr);
  
  while (args[param_index])
  {
    file_to_remove = current_directory->first_file;
    success = 0;
    
    while (file_to_remove)
    {
      if (!strcmp(file_to_remove->filename, args[param_index]))
      {
        success = 1;
        if (is_file(file_to_remove))
        {
          memset(file_to_remove->filename, 0, sizeof(file_to_remove->filename));
          if (file_to_remove->content)
            free(file_to_remove->content);
          unlink_file(file_to_remove);
        }
        else
        {
          fprintf(stderr, "mybash: '%s': Is a directory\n", args[param_index]);
        }
        break;
      }
      file_to_remove = file_to_remove->next_file;
    }
    
    if (success != 1)
      fprintf(stderr, "mybash: '%s': No such file or directory\n", args[param_index]);
    ++param_index;
  }
  
  return 1LL;
}

echo Command

__int64 command_echo(char **args)
{
  MemoryBlock *file_to_write; // rbx
  MemoryBlock *target_file; // rbx
  int param_index; // [rsp+14h] [rbp-3Ch]
  int i; // [rsp+14h] [rbp-3Ch]
  int content_length; // [rsp+18h] [rbp-38h]
  int last_param_index; // [rsp+1Ch] [rbp-34h]
  int input_length; // [rsp+24h] [rbp-2Ch]
  MemoryBlock *current_file; // [rsp+28h] [rbp-28h] BYREF
  const char *last_param; // [rsp+30h] [rbp-20h]
  unsigned __int64 v11; // [rsp+38h] [rbp-18h]

  v11 = __readfsqword(0x28u);
  param_index = 0;
  
  if (args[1])
  {
    do
      ++param_index;
    while (args[param_index]);
    
    last_param_index = param_index - 1;
    
    if (no_output_to_file(args[param_index - 2]))
    {
      for (i = 1; i < last_param_index; ++i)
        printf(args[i]);
      puts(args[last_param_index]);
      return 1LL;
    }
    else
    {
      current_file = current_directory->first_file;
      last_param = args[last_param_index];
      
      if (file_exists(&current_file, last_param) != 1)
      {
        fprintf(stderr, "mybash: %s: No such file\n", last_param);
        return 1LL;
      }
      else if (is_file(current_file))
      {
        if (current_file->content)
        {
          content_length = get_content_length(current_file->content);
        }
        else
        {
          content_length = 0x1F0;
          file_to_write = current_file;
          file_to_write->content = malloc(0x1F0uLL);
          if (!current_file->content)
            _exit(0);
        }
        
        input_length = strlen(args[last_param_index - 2]);
        while (input_length > content_length)
          content_length += 0x1F0;
        
        if (content_length > get_content_length(current_file->content))
        {
          target_file = current_file;
          target_file->content = realloc(current_file->content, content_length);
        }
        
        strncpy(current_file->content, args[last_param_index - 2], input_length);
        return 1LL;
      }
      else
      {
        fprintf(stderr, "mybash: %s: Is a directory\n", last_param);
        return 1LL;
      }
    }
  }
  else
  {
    putchar(10);
    return 1LL;
  }
}

cd Command

__int64 command_cd(char **args)
{
  size_t path_length; // rbx
  size_t filename_length; // rax
  int current_dir_name_length; // [rsp+18h] [rbp-38h]
  const char *path_component; // [rsp+20h] [rbp-30h]
  MemoryBlock *current_file; // [rsp+28h] [rbp-28h]
  char delimiter[2]; // [rsp+36h] [rbp-1Ah] BYREF
  unsigned __int64 v8; // [rsp+38h] [rbp-18h]

  v8 = __readfsqword(0x28u);
  
  if (args[1])
  {
    if (args[2])
    {
      fwrite("mybash: too many arguments\n", 1uLL, 0x1BuLL, stderr);
    }
    else
    {
      strcpy(delimiter, "/");
      
      for (path_component = strtok(args[1], delimiter); path_component; 
           path_component = strtok(0LL, delimiter))
      {
        if (strcmp(path_component, "."))
        {
          if (!strcmp(path_component, ".."))
          {
            if (current_directory->parent)
            {
              current_dir_name_length = strlen(current_directory->filename);
              current_path[(strlen(current_path) - 1 - current_dir_name_length)] = 0;
              current_directory = current_directory->parent;
            }
          }
          else
          {
            current_file = current_directory->first_file;
            
            if (folder_exists(current_file, path_component))
            {
              fprintf(stderr, "mybash: %s: No such file or directory\n", path_component);
              return 1LL;
            }
            
            while (current_file && strcmp(current_file->filename, path_component))
              current_file = current_file->next_file;
            
            if (!is_directory(current_file))
            {
              fwrite("something wrong happened\n", 1uLL, 0x19uLL, stderr);
              return 1LL;
            }
            
            current_directory = current_file;
            path_length = strlen(current_path);
            
            if (path_length + strlen(current_file->filename) + 1 <= 0x4F)
            {
              filename_length = strlen(current_file->filename);
              strncat(current_path, current_file->filename, filename_length);
              *&current_path[strlen(current_path)] = 47;
            }
          }
        }
      }
    }
  }
  else
  {
    fwrite("mybash: expected argument\n", 1uLL, 0x1AuLL, stderr);
  }
  
  return 1LL;
}

Tags: IDA Pro reverse engineering Pwn CTF x86-64 assembly

Posted on Mon, 26 Oct 2026 16:55:39 +0000 by alco19357