<div dir="ltr">Group,<div><br></div><div><br></div><div>I get about two nightly crashes in the latest opensips head this has been happening consistently for about a week and half now. All crashes are identical to:</div><div>
<p class=""><span class="">Using host libthread_db library "/lib64/libthread_db.so.1".</span></p>
<p class=""><span class="">Core was generated by `/sbin/opensips -P /var/run/opensips.pid -u opensips -g opensips -M 1024 -f /etc'.</span></p>
<p class=""><span class="">Program terminated with signal 11, Segmentation fault.</span></p>
<p class=""><span class="">#0 0x0000000000512512 in fm_remove_free (n=0x7f384a06b680, qm=0x7f3849fc8010) at mem/f_malloc.c:187</span></p>
<p class=""><span class="">187<span class=""> </span><span class=""> </span>*pf=n->u.nxt_free;</span></p>
<p class=""><span class="">Missing separate debuginfos, use: debuginfo-install cyrus-sasl-lib-2.1.26-20.el7_2.x86_64 glibc-2.17-106.el7_2.4.x86_64 gmp-6.0.0-12.el7_1.x86_64 gnutls-3.3.8-14.el7_2.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.13.2-12.el7_2.x86_64 libcom_err-1.42.9-7.el7.x86_64 libcurl-7.29.0-25.el7.centos.x86_64 libffi-3.0.13-16.el7.x86_64 libgcc-4.8.5-4.el7.x86_64 libgcrypt-1.5.3-12.el7_1.1.x86_64 libgpg-error-1.12-3.el7.x86_64 libidn-1.28-4.el7.x86_64 libmicrohttpd-0.9.33-2.el7.x86_64 librabbitmq-0.5.2-1.el7.x86_64 libselinux-2.2.2-6.el7.x86_64 libssh2-1.4.3-10.el7_2.1.x86_64 libstdc++-4.8.5-4.el7.x86_64 libtasn1-3.8-2.el7.x86_64 nettle-2.7.1-4.el7.x86_64 nspr-4.11.0-1.el7_2.x86_64 nss-3.21.0-9.el7_2.x86_64 nss-softokn-freebl-3.16.2.3-14.2.el7_2.x86_64 nss-util-3.21.0-2.2.el7_2.x86_64 openldap-2.4.40-9.el7_2.x86_64 openssl-libs-1.0.1e-51.el7_2.4.x86_64 p11-kit-0.20.7-3.el7.x86_64 pcre-8.32-15.el7.x86_64 trousers-0.3.13-1.el7.x86_64 xz-libs-5.1.2-12alpha.el7.x86_64 zlib-1.2.7-15.el7.x86_64</span></p>
<p class=""><span class="">(gdb) bt full</span></p>
<p class=""><span class="">#0 0x0000000000512512 in fm_remove_free (n=0x7f384a06b680, qm=0x7f3849fc8010) at mem/f_malloc.c:187</span></p>
<p class=""><span class=""> pf = 0x0</span></p>
<p class=""><span class=""> hash = 2051</span></p>
<p class=""><span class="">#1 fm_malloc (qm=0x7f3849fc8010, size=size@entry=65592) at mem/f_malloc.c:415</span></p>
<p class=""><span class=""> frag = 0x7f384a06b680</span></p>
<p class=""><span class=""> n = <optimized out></span></p>
<p class=""><span class=""> hash = 2051</span></p>
<p class=""><span class=""> __FUNCTION__ = "fm_malloc"</span></p>
<p class=""><span class="">#2 0x00007f383e018652 in tcp_handle_req (_max_msg_chunks=<optimized out>, con=<optimized out>, req=<optimized out>) at ../../net/proto_tcp/tcp_common.h:459</span></p>
<p class=""><span class=""> local_rcv = {src_ip = {af = 760673648, len = 32766, u = {addrl = {16, 20}, addr32 = {16, 0, 20, 0}, addr16 = {16, 0, 0, 0, 20, 0, 0, 0}, addr = "\020\000\000\000\000\000\000\000\024\000\000\000\000\000\000"}}, dst_ip = {af = 1, len = 1, u = {addrl = {</span></p>
<p class=""><span class=""> 139878494896234, 5869476}, addr32 = {106, 32568, 5869476, 0}, addr16 = {106, 0, 32568, 0, 36772, 89, 0, 0}, addr = "j\000\000\000\070\177\000\000\244\217Y\000\000\000\000"}}, src_port = 0, dst_port = 0, proto = 0, proto_reserved1 = 0, </span></p>
<p class=""><span class=""> proto_reserved2 = 0, src_su = {s = {sa_family = 62576, sa_data = "V-\376\177\000\000\001\000\000\000\000\000\000"}, sin = {sin_family = 62576, sin_port = 11606, sin_addr = {s_addr = 32766}, sin_zero = "\001\000\000\000\000\000\000"}, sin6 = {</span></p>
<p class=""><span class=""> sin6_family = 62576, sin6_port = 11606, sin6_flowinfo = 32766, sin6_addr = {__in6_u = {__u6_addr8 = "\001\000\000\000\000\000\000\000\200\364V-\376\177\000", __u6_addr16 = {1, 0, 0, 0, 62592, 11606, 32766, 0}, __u6_addr32 = {1, 0, 760673408, 32766}}}, </span></p>
<p class=""><span class=""> sin6_scope_id = 24}}, bind_address = 0x4a03e63000000001}</span></p>
<p class=""><span class=""> msg_buf = <optimized out></span></p>
<p class=""><span class=""> msg_len = <optimized out></span></p>
<p class=""><span class=""> c = <optimized out></span></p>
<p class=""><span class=""> size = <optimized out></span></p>
<p class=""><span class="">#3 tls_read_req (con=0x7f38481398c0, bytes_read=0x7ffe2d56f550) at proto_tls.c:441</span></p>
<p class=""><span class=""> bytes = <optimized out></span></p>
<p class=""><span class=""> total_bytes = 0</span></p>
<p class=""><span class=""> req = <optimized out></span></p>
<p class=""><span class=""> __FUNCTION__ = "tls_read_req"</span></p>
<p class=""><span class="">#4 0x000000000059c478 in handle_io (fm=fm@entry=0x7f384a03e630, idx=idx@entry=3, event_type=event_type@entry=1) at net/net_tcp_proc.c:205</span></p>
<p class=""><span class=""> ret = 0</span></p>
<p class=""><span class=""> n = <optimized out></span></p>
<p class=""><span class=""> con = 0x7f38481398c0</span></p>
<p class=""><span class=""> s = 106</span></p>
<p class=""><span class=""> rw = <optimized out></span></p>
<p class=""><span class=""> resp = <optimized out></span></p>
<p class=""><span class=""> response = {139879704139968, 1}</span></p>
<p class=""><span class=""> __FUNCTION__ = "handle_io"</span></p>
<p class=""><span class="">#5 0x000000000059daee in io_wait_loop_epoll (h=<optimized out>, t=<optimized out>, repeat=<optimized out>) at net/../io_wait_loop.h:221</span></p>
<p class=""><span class=""> ret = <optimized out></span></p>
<p class=""><span class=""> e = <optimized out></span></p>
<p class=""><span class=""> n = 1</span></p>
<p class=""><span class=""> r = 4</span></p>
<p class=""><span class="">#6 tcp_worker_proc (unix_sock=<optimized out>) at net/net_tcp_proc.c:312</span></p>
<p class=""><span class=""> __FUNCTION__ = "tcp_worker_proc"</span></p>
<p class=""><span class="">#7 0x00000000005a80c3 in tcp_start_processes (chd_rank=chd_rank@entry=0x841360 <chd_rank.11009>, startup_done=startup_done@entry=0x7f3848136888) at net/net_tcp.c:1758</span></p>
<p class=""><span class=""> r = 2</span></p>
<p class=""><span class=""> reader_fd = {98, 100}</span></p>
<p class=""><span class=""> pid = 0</span></p>
<p class=""><span class=""> load_p = 0x7f38481374e8</span></p>
<p class=""><span class=""> __FUNCTION__ = "tcp_start_processes"</span></p>
<p class=""><span class="">#8 0x0000000000419f05 in main_loop () at main.c:677</span></p>
<p class=""><span class=""> startup_done = 0x7f3848136888</span></p>
<p class=""><span class=""> chd_rank = 15</span></p>
<p class=""><span class="">#9 main (argc=<optimized out>, argv=<optimized out>) at main.c:1258</span></p>
<p class=""><span class=""> cfg_stream = <optimized out></span></p>
<p class=""><span class=""> c = <optimized out></span></p>
<p class=""><span class=""> r = <optimized out></span></p>
<p class=""><span class=""> tmp = 0x7ffe2d56ff65 ""</span></p>
<p class=""><span class=""> tmp_len = <optimized out></span></p>
<p class=""><span class=""> port = <optimized out></span></p>
<p class=""><span class=""> proto = <optimized out></span></p>
<p class=""><span class=""> protos_no = <optimized out></span></p>
<p class=""><span class=""> options = 0x5daf00 "f:cCm:M:b:l:n:N:rRvdDFETSVhw:t:u:g:P:G:W:o:"</span></p>
<p class=""><span class=""> ret = -1</span></p>
<p class=""><span class=""> seed = 3936233749</span></p>
<p class=""><span class=""> rfd = <optimized out></span></p>
<p class=""><span class=""> __FUNCTION__ = "main"</span></p><p class=""><span class=""><br></span></p><p class=""><span class=""><br></span></p><p class=""><span class="">Please advise if anything else is needed.</span></p></div></div>