[OpenSIPS-Users] segfault in mem/f_malloc.c

[Bobby Smith]

This is on revision 7081.

It seems to be fairly consistently happening around free_dlg_dlg in
dlg_hash.c which is triggering it -- we have several cores showing this
behavior.

What are the next steps for debugging this?  This is on a production machine
and we haven't been able to reproduce it elsewhere, so we are somewhat
sensitive to load issues while trying to troubleshoot this.

Thanks.



Program terminated with signal 11, Segmentation fault.
#0  0x00000000004a2bcf in fm_insert_free (qm=0x2b875eba4000, p=<value
optimized out>) at mem/f_malloc.c:155
155 (*f)->prev = &(frag->u.nxt_free);
(gdb) bt full
#0  0x00000000004a2bcf in fm_insert_free (qm=0x2b875eba4000, p=<value
optimized out>) at mem/f_malloc.c:155
        f = 0x2b875eba4070
        hash = 3
#1  fm_free (qm=0x2b875eba4000, p=<value optimized out>) at
mem/f_malloc.c:460
        f = 0x2b875f41be40
        n = <value optimized out>
        __FUNCTION__ = "fm_free"
#2  0x00002b875e14dcde in free_dlg_dlg () at dlg_hash.c:168
        i = 0
#3  destroy_dlg_table () at dlg_hash.c:234
        dlg = 0x2b875f319a80
        i = 5
#4  0x00002b875e13a651 in mod_destroy () at dialog.c:742
No locals.
#5  0x0000000000476ff4 in destroy_modules () at sr_module.c:370
        t = 0x785328
        foo = 0x785258
#6  0x00000000004252e0 in cleanup (show_status=1) at main.c:336
No locals.
#7  0x00000000004261fb in handle_sigs () at main.c:533
        chld = 0
        chld_status = 139
        i = <value optimized out>
        do_exit = 1
        __FUNCTION__ = "handle_sigs"
#8  0x000000000042a5a9 in main_loop (argc=3, argv=0x4) at main.c:913
        i = 4
        pid = <value optimized out>
        si = 0x0
        startup_done = 0x0
        chd_rank = 16
        __FUNCTION__ = "main_loop"
#9  main (argc=3, argv=0x4) at main.c:1388
        cfg_log_stderr = <value optimized out>
        cfg_stream = 0x831e010
        c = <value optimized out>
        r = <value optimized out>
        tmp = 0x4fc447 "H\215\005\262\333#"
        tmp_len = <value optimized out>
        port = <value optimized out>
        proto = <value optimized out>
        ret = <value optimized out>
        seed = 586369539
        rfd = <value optimized out>
        __FUNCTION__ = "main"
(gdb)


(gdb) frame 0
#0  0x00000000004a2bcf in fm_insert_free (qm=0x2b875eba4000, p=<value
optimized out>) at mem/f_malloc.c:155
155 (*f)->prev = &(frag->u.nxt_free);
(gdb) list
150
151 /*insert it here*/
152 frag->prev = f;
153 frag->u.nxt_free=*f;
154 if( *f )
155 (*f)->prev = &(frag->u.nxt_free);
156
157 *f=frag;
158 qm->free_hash[hash].no++;
159 free_plus(qm , frag->size);
(gdb) info locals
f = 0x2b875eba4070
hash = 3
(gdb) print *f
$1 = (struct fm_frag *) 0x2b865f235290
(gdb) print *(*f)
Cannot access memory at address 0x2b865f235290
(gdb) frame 1
#1  fm_free (qm=0x2b875eba4000, p=<value optimized out>) at
mem/f_malloc.c:460
460 fm_insert_free(qm, f);
(gdb) print f
$2 = (struct fm_frag *) 0x2b875f41be40
(gdb) list
455 goto join;
456 }
457
458 no_join:
459
460 fm_insert_free(qm, f);
461 }
462
463
464 #ifdef DBG_F_MALLOC
(gdb)


And a second crash here a few hours after restarting from above:

Core was generated by `/usr/local/opensips/sbin/opensips -u opensips'.
Program terminated with signal 11, Segmentation fault.
#0  0x00000000004a2bcf in fm_insert_free (qm=0x2aca30c14000, p=<value
optimized out>) at mem/f_malloc.c:155
155 (*f)->prev = &(frag->u.nxt_free);
(gdb) bt full
#0  0x00000000004a2bcf in fm_insert_free (qm=0x2aca30c14000, p=<value
optimized out>) at mem/f_malloc.c:155
        f = 0x2aca30c14070
        hash = 3
#1  fm_free (qm=0x2aca30c14000, p=<value optimized out>) at
mem/f_malloc.c:460
        f = 0x2aca315c1718
        n = <value optimized out>
        __FUNCTION__ = "fm_free"
#2  0x00002aca301bdcde in free_dlg_dlg () at dlg_hash.c:168
        i = 0
#3  destroy_dlg_table () at dlg_hash.c:234
        dlg = 0x2aca31692c70
        i = 0
#4  0x00002aca301aa651 in mod_destroy () at dialog.c:742
No locals.
#5  0x0000000000476ff4 in destroy_modules () at sr_module.c:370
        t = 0x785328
        foo = 0x785258
#6  0x00000000004252e0 in cleanup (show_status=1) at main.c:336
No locals.
#7  0x00000000004261fb in handle_sigs () at main.c:533
        chld = 0
        chld_status = 139
        i = <value optimized out>
        do_exit = 1
        __FUNCTION__ = "handle_sigs"
#8  0x000000000042a5a9 in main_loop (argc=3, argv=0x4) at main.c:913
        i = 4
        pid = <value optimized out>
        si = 0x0
        startup_done = 0x0
        chd_rank = 16
        __FUNCTION__ = "main_loop"
#9  main (argc=3, argv=0x4) at main.c:1388
        cfg_log_stderr = <value optimized out>
        cfg_stream = 0x14cfb010
        c = <value optimized out>
        r = <value optimized out>
        tmp = 0x4fc447 "H\215\005\262\333#"
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.opensips.org/pipermail/users/attachments/20110214/f34c58af/attachment-0001.htm>