Debugging

This is a general page on debugging tools, techniques, tips, etc.

Debuggers environment

OllyDbg

OllyDbg is a 32-bit assembler level analysing debugger for Microsoft® Windows®.

Hopper Disassembler

Hopper is a reverse engineering tool for OS X and Linux, that lets you disassemble, decompile and debug your 32/64bits Intel Mac, Linux, Windows and iOS executables.

Linux tools

addr2line

See addr2line page.

GDB and GDB front-ends

See GDB page.

Profiling

• http://poormansprofiler.org/
• https://github.com/yoshinorim/quickstack
• https://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html

ftrace

• https://jvns.ca/blog/2017/03/19/getting-started-with-ftrace/

C/C++ - Debugging with gcc

backtrace, backtrace_symbols, backtrace_symbols_fd

See manual page, or gcc manual.

int     backtrace            (void **      buffer, int size)
char ** backtrace_symbols    (void *const *buffer, int size)
void    backtrace_symbols_fd (void *const *buffer, int size, int fd)

Object and executables need to be built (i.e. compiled and linked) with gcc -g -rdynamic (also -fvisibility must either be default, protected or not set at all)!

gcc -O0 -g -rdynamic main.c -o myprogram

Short example:

#include <stdio.h>
#include <execinfo.h>
#include <signal.h>
#include <stdlib.h>

void handler(int sig) {
  void *array[10];
  int size;

  size = backtrace(array, 10);                  // get void*'s for all entries on the stack

  fprintf(stderr, "Error: signal %d:\n", sig);  // print out all the frames to stderr
  backtrace_symbols_fd(array, size, 2);
  exit(1);
}

void baz() {
  int *foo = (int*)-1;                          // make a bad pointer
  printf("%d\n", *foo);                         // causes segfault
}

void bar() { baz(); }
void foo() { bar(); }

int main(int argc, char **argv) {
  signal(SIGSEGV, handler);                     // install our handler
  foo();                                        // this will call foo, bar, and baz.  baz segfaults.
  return 0;
}

It can also be used to print the symbol name of any function of which we know the address (a bit like SymFromAddr on windows):

#include <stdio.h>
#include <execinfo.h>

void foo(void) {
    printf("foo\n");
}

int main(int argc, char *argv[]) {
    void    *funptr = &foo;

    backtrace_symbols_fd(&funptr, 1, 1);

    return 0;
}

A very extensive answer on StackOverflow:

• Using backtrace:

main.c ([{{#filelink: main.c}} download])

{{#fileanchor: main.c}}

#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#ifndef __USE_GNU
#define __USE_GNU
#endif

#include <execinfo.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ucontext.h>
#include <unistd.h>

/* This structure mirrors the one found in /usr/include/asm/ucontext.h */
typedef struct _sig_ucontext {
    unsigned long     uc_flags;
    struct ucontext   *uc_link;
    stack_t           uc_stack;
    struct sigcontext uc_mcontext;
    sigset_t          uc_sigmask;
} sig_ucontext_t;

void crit_err_hdlr(int sig_num, siginfo_t * info, void * ucontext)
{
    void *             array[50];
    void *             caller_address;
    char **            messages;
    int                size, i;
    sig_ucontext_t *   uc;

    uc = (sig_ucontext_t *)ucontext;

    /* Get the address at the time the signal was raised from the EIP (x86) */
    caller_address = (void *) uc->uc_mcontext.eip;   

    fprintf(stderr, "signal %d (%s), address is %p from %p\n", 
        sig_num, strsignal(sig_num), info->si_addr, 
        (void *)caller_address);

    size = backtrace(array, 50);

    /* overwrite sigaction with caller's address */
    array[1] = caller_address;

    messages = backtrace_symbols(array, size);

    /* skip first stack frame (points here) */
    for (i = 1; i < size && messages != NULL; ++i)
    {
        fprintf(stderr, "[bt]: (%d) %s\n", i, messages[i]);
    }

    free(messages);

    exit(EXIT_FAILURE);
}

int crash()
{
    char * p = NULL;
    *p = 0;
    return 0;
}

int foo4()
{
    crash();
    return 0;
}

int foo3()
{
    foo4();
    return 0;
}

int foo2()
{
    foo3();
    return 0;
}

int foo1()
{
    foo2();
    return 0;
}

int main(int argc, char ** argv)
{
    struct sigaction sigact;

    sigact.sa_sigaction = crit_err_hdlr;
    sigact.sa_flags = SA_RESTART | SA_SIGINFO;

    if (sigaction(SIGSEGV, &sigact, (struct sigaction *)NULL) != 0)
    {
        fprintf(stderr, "error setting signal handler for %d (%s)\n",
        SIGSEGV, strsignal(SIGSEGV));

        exit(EXIT_FAILURE);
    }

    foo1();

    exit(EXIT_SUCCESS);
}

• Demangling symbols in C++ by calling __cxa_demangle (like is done in tool c++filt):

main.c ([{{#filelink: main.c}} download])

{{#fileanchor: main.c}}

void crit_err_hdlr(int sig_num, siginfo_t * info, void * ucontext)
{
    sig_ucontext_t * uc = (sig_ucontext_t *)ucontext;

    void * caller_address = (void *) uc->uc_mcontext.eip; // x86 specific

    std::cerr << "signal " << sig_num 
              << " (" << strsignal(sig_num) << "), address is " 
              << info->si_addr << " from " << caller_address 
              << std::endl << std::endl;

    void * array[50];
    int size = backtrace(array, 50);

    array[1] = caller_address;

    char ** messages = backtrace_symbols(array, size);    

    // skip first stack frame (points here)
    for (int i = 1; i < size && messages != NULL; ++i)
    {
        char *mangled_name = 0, *offset_begin = 0, *offset_end = 0;

        // find parantheses and +address offset surrounding mangled name
        for (char *p = messages[i]; *p; ++p)
        {
            if (*p == '(') 
            {
                mangled_name = p; 
            }
            else if (*p == '+') 
            {
                offset_begin = p;
            }
            else if (*p == ')')
            {
                offset_end = p;
                break;
            }
        }

        // if the line could be processed, attempt to demangle the symbol
        if (mangled_name && offset_begin && offset_end && 
            mangled_name < offset_begin)
        {
            *mangled_name++ = '\0';
            *offset_begin++ = '\0';
            *offset_end++ = '\0';

            int status;
            char * real_name = abi::__cxa_demangle(mangled_name, 0, 0, &status);

            // if demangling is successful, output the demangled function name
            if (status == 0)
            {    
                std::cerr << "[bt]: (" << i << ") " << messages[i] << " : " 
                          << real_name << "+" << offset_begin << offset_end 
                          << std::endl;

            }
            // otherwise, output the mangled function name
            else
            {
                std::cerr << "[bt]: (" << i << ") " << messages[i] << " : " 
                          << mangled_name << "+" << offset_begin << offset_end 
                          << std::endl;
            }
            free(real_name);
        }
        // otherwise, print the whole line
        else
        {
            std::cerr << "[bt]: (" << i << ") " << messages[i] << std::endl;
        }
    }
    std::cerr << std::endl;

    free(messages);

    exit(EXIT_FAILURE);
}

Getting function name from its address

• TODO: There is maybe an alternate solution using dlopen, which would let an application to browse its own symbols.
• TODO: Check whether we can force export of (debug) symbols even if declared static.

The following program show how to use the backtrace_symbols function to get the name of a function given its address.

The program must be compiled (and linked) with gcc options -g -rdynamic (also option -fvisibility must either be default, protected or not set at all):

gcc -g -rdynamic -O0 main.c -o symfromaddr

main.c ([{{#filelink: main.c}} download])

{{#fileanchor: main.c}}

#include <stdio.h>
#include <execinfo.h>
#include <stdlib.h>
#include <assert.h>

typedef void (*f_ptr)(void);


void callbackfunction(void)
{
    printf("callbackfunction\n");
}

void symfromaddr(char **sym, void * fptr, void **buftofree)
{
    char * p;

    (*buftofree)=backtrace_symbols(&fptr, 1);
    assert(*buftofree!=NULL);

    //Isolate fctname in './path/with/(parens)/pgm(fctname+0) [0x1234abcd]'
    (*sym)=p=((char **)buftofree)[0];
    while( (*p) && ('+' != (*p)) )
    {
        if( '(' == *(p++) )
        {
            (*sym)=p;
        }
    }
    if(!*p)
      (*sym)=p;
    *p = 0;
}


int main(int argc, char **argv) {
    char *sym;
    void *buftofree;

    symfromaddr(&sym,callbackfunction,&buftofree);
    printf("func symbol is '%s'\n",sym);
    free(buftofree);

    return 0;
}

A less efficient but more convenient version:

main.c ([{{#filelink: main.c}} download])

{{#fileanchor: main.c}}

#include <stdio.h>
#include <execinfo.h>
#include <stdlib.h>
#include <assert.h>
#include <string.h>

typedef void (*f_ptr)(void);


void callbackfunction(void)
{
    printf("callbackfunction\n");
}

char * symfromaddr(char *sym, void * fptr, unsigned n)
{
    char *beg,*end;
    char ** strings;

    strings=backtrace_symbols(&fptr, 1);
    assert(strings!=NULL);

    beg=end=strings[0];
    while( (*end) && ('+' != (*end)) )
    {
        if( '(' == *(end++) )
        {
            beg=end;
        }
    }

    sym[0]=0;
    if(*end) {
      *end = 0;
      strncpy(sym,beg,n);
      if (n>0)
          sym[n-1]=0;
    }

    free(strings);

    return sym;
}


int main(int argc, char **argv) {
    char sym[20];

    printf("func symbol is '%s'\n",symfromaddr(sym,callbackfunction,sizeof(sym)));

    return 0;
}

libunwind (non-gnu)

• See http://www.nongnu.org/libunwind/docs.html (see also here for example)

Debugging pthreads

GDB:

• Debugging Programs with Multiple Threads
• GDB: Stopping and starting multi-thread programs
• GDB/MI: Threads commands

DDD:

• Examining Threads