Thursday, July 18, 2013

Linux pthread test app with lldb

Testing some multithreaded debugging with lldb. We've got four issues to start looking at so far...

1. Running "gdb -- blah", "b nanosleep", "r", will result in breaking on the nanosleep call.
Running "lldb -- blah", "b nanosleep", "r", will result in breaking in the 'ret' instruction of the nanosleep call. So the break doesn't happen until after the sleep period.

2. Doing a "gdb -- blah", "b main", "r", "b 71", "c", "info threads" will result in this on gdb:

(gdb) info threads
  Id   Target Id         Frame
  3    Thread 0x7ffff65e2700 (LWP 5353) "thread_1" 0x00007ffff6ea384d in nanosleep () at ../sysdeps/unix/syscall-template.S:82
  2    Thread 0x7ffff6de3700 (LWP 5350) "thread_0" 0x00007ffff6ea384d in nanosleep () at ../sysdeps/unix/syscall-template.S:82
* 1    Thread 0x7ffff7fd1740 (LWP 5220) "mainthrd" main (argc=1, argv=0x7fffffffdd08) at /home/mikesart/data/src/blah_pthreads/blah.cpp:71

This on lldb:

(lldb) thread list
Process 5520 stopped
* thread #1: tid = 0x1590, 0x000000000040113f blah`main(argc=1, argv=0x00007fff4ef17f28) + 495 at blah.cpp:71, name = 'blah, stop reason = breakpoint 2.1
  thread #2: tid = 0x15ff, 0x00007f92502d584d libc.so.6`__nanosleep + 45 at syscall-template.S:82, name = 'mainthrd
  thread #3: tid = 0x1600, 0x00007f92502d584d libc.so.6`__nanosleep + 45 at syscall-template.S:82, name = 'mainthrd

The TIDs are hex and the thread names are wrong.

3. On gdb, "thread apply all" gives us this:

(gdb) thread apply all bt
Thread 3 (Thread 0x7ffff65e2700 (LWP 5353)):
#0  0x00007ffff6ea384d in nanosleep () at ../sysdeps/unix/syscall-template.S:82
#1  0x00007ffff6ea36ec in __sleep (seconds=0) at ../sysdeps/unix/sysv/linux/sleep.c:138
#2  0x0000000000400f28 in thread_proc (arg=0x1) at /home/mikesart/data/src/blah_pthreads/blah.cpp:37
#3  0x00007ffff79c0e9a in start_thread (arg=0x7ffff65e2700) at pthread_create.c:308
#4  0x00007ffff6ed7ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()
Thread 2 (Thread 0x7ffff6de3700 (LWP 5350)):
#0  0x00007ffff6ea384d in nanosleep () at ../sysdeps/unix/syscall-template.S:82
#1  0x00007ffff6ea36ec in __sleep (seconds=0) at ../sysdeps/unix/sysv/linux/sleep.c:138
#2  0x0000000000400f28 in thread_proc (arg=0x0) at /home/mikesart/data/src/blah_pthreads/blah.cpp:37
#3  0x00007ffff79c0e9a in start_thread (arg=0x7ffff6de3700) at pthread_create.c:308
#4  0x00007ffff6ed7ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()
Thread 1 (Thread 0x7ffff7fd1740 (LWP 5220)):
#0  main (argc=1, argv=0x7fffffffdd08) at /home/mikesart/data/src/blah_pthreads/blah.cpp:71

On lldb, this (no backtraces for sleeping threads):

(lldb) bt all
* thread #1: tid = 0x1590, 0x000000000040113f blah`main(argc=1, argv=0x00007fff4ef17f28) + 495 at blah.cpp:78, name = 'blah, stop reason = breakpoint 2.1
    frame #0: 0x000000000040113f blah`main(argc=1, argv=0x00007fff4ef17f28) + 495 at blah.cpp:71
    frame #1: 0x00007f925023776d libc.so.6`__libc_start_main(main=0x0000000000400f50, argc=1, ubp_av=0x00007fff4ef17f28, init=<unavailable>, fini=<unavailable>, rtld_fini=<unavailable>, stack_end=0x00007fff4ef17f18) + 237 at libc-start.c:226
    frame #2: 0x0000000000400b99 blah`_start + 41
  thread #2: tid = 0x15ff, 0x00007f92502d584d libc.so.6`__nanosleep + 45 at syscall-template.S:82, name = 'mainthrd
    frame #0: 0x00007f92502d584d libc.so.6`__nanosleep + 45 at syscall-template.S:82
  thread #3: tid = 0x1600, 0x00007f92502d584d libc.so.6`__nanosleep + 45 at syscall-template.S:82, name = 'mainthrd
    frame #0: 0x00007f92502d584d libc.so.6`__nanosleep + 45 at syscall-template.S:82

4. gdb has these excellent thread-events (controllable via set print thread-events). Nothing like this in lldb yet.
      [New Thread 0x7ffff6de3700 (LWP 19270)]
      [Thread 0x7ffff6de3700 (LWP 19270) exited]

Here is the test source.

───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
   1 #include <stdio.h>
   2 #include <string.h>
   3 #include <pthread.h>
   4 #include <stdlib.h>
   5 #include <unistd.h>
   6 #include <sys/syscall.h>
   7 
   8 __thread pid_t g_tls = -1;
   9 __thread char g_threadname[32];
  10 
  11 pid_t gettid()
  12 {
  13     return (pid_t)syscall(SYS_gettid);
  14 }
  15 
  16 void logf(const char *format, ...)
  17 {
  18     va_list args;
  19     char buf[1024];
  20 
  21     snprintf(buf, sizeof(buf), "'%s' [#%d LWP:%d 0x%lx] %s", g_threadname, g_tls, gettid(), pthread_self(), format);
  22 
  23     va_start (args, format);
  24     vprintf (buf, args);
  25     va_end (args);
  26 }
  27 
  28 void *thread_proc(void *arg)
  29 {
  30     g_tls = (int)(intptr_t)arg;
  31 
  32     logf("pthread_setname_np('%s')\n", g_threadname);
  33     snprintf(g_threadname, sizeof(g_threadname), "thread_%d", g_tls);
  34     pthread_setname_np(pthread_self(), g_threadname);
  35 
  36     logf("sleep(5)\n");
  37     sleep(5);
  38 
  39     pid_t tid = gettid();
  40     logf("pthread_exit(%d)\n", tid);
  41     pthread_exit((void *)(intptr_t)tid);
  42     return 0;
  43 }
  44 
  45 int main(int argc, char *argv[])
  46 {
  47     pthread_t threadids[256];
  48     static const size_t max_threads = sizeof(threadids) / sizeof(threadids[0]);
  49 
  50     size_t num_threads = (argc > 1) ? atoi(argv[1]) : 2;
  51     if (num_threads < 2)
  52         num_threads = 2;
  53     else if (num_threads > max_threads)
  54         num_threads = max_threads;
  55 
  56     snprintf(g_threadname, sizeof(g_threadname), "mainthrd");
  57     pthread_setname_np(pthread_self(), g_threadname);
  58 
  59     printf("num_threads:%zu\n", num_threads);
  60 
  61     for(size_t i = 0; i < num_threads; i++)
  62     {
  63         int err = pthread_create(&(threadids[i]), NULL, &thread_proc, (void *)(intptr_t)i);
  64         logf("pthread_create:%d (%s) pthread_t:%lx\n", err, strerror(err), threadids[i]);
  65     }
  66 
  67     sleep(1);
  68 
  69     for(size_t i = 0; i < num_threads; i++)
  70     {
  71         logf("Waiting for thread #%zu\n", i);
  72 
  73         void *status = NULL;
  74         int rc = pthread_join(threadids[i], &status);
  75         logf("Thread #%zu rc:%d status:%d\n", i, rc, (int)(intptr_t)status);
  76     }
  77 
  78     printf("done.\n");
  79     return 0;
  80 } 

1 comment:

  1. Mike - FYI I ported your test code to FreeBSD here:
    https://github.com/emaste/snippets/blob/master/mikesart_pthread.c

    ReplyDelete