[OpenSIPS-Users] 3.2.3 TLS issue
Gregory Massel
greg at switchtel.co.za
Thu Dec 9 00:46:38 EST 2021
Herewith some backtraces:
#0 __strlen_sse2 () at ../sysdeps/x86_64/multiarch/../strlen.S:120
No locals.
#1 0x00007f3e6c2b25d3 in _IO_vfprintf_internal (s=s at entry=0x557c28170060,
format=format at entry=0x557c27af4378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", ap=0x7ffce94b5188)
at vfprintf.c:1643
len = <optimized out>
string_malloced = 0
string = 0x1343c6c031e289 <error: Cannot access memory at address 0x1343c6c031e289>
width = <optimized out>
signed_number = <optimized out>
is_short = 0
spec = 115 's'
ptr = <optimized out>
ptr = <optimized out>
outc = <optimized out>
step0_jumps = {0, 160, 64, 1664, 1576, 1464, 1360, 2216, 2912, 208, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256,
2624, -1344, -1344, 2536}
base = <optimized out>
pad = <optimized out>
offset = <optimized out>
offset = <optimized out>
step1_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, -1344, -1344, 0}
args_value = <optimized out>
is_negative = <optimized out>
number = <optimized out>
use_outdigits = <optimized out>
step2_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, -1344, -1344, 0}
alt = 0
left = 0
showsign = <optimized out>
group = 0
is_char = 0
step3a_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 0, 0, 0, -760, 3024, 3000, 3104, 2128, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0}
the_arg = {pa_wchar = 2037198692 L'\x796d2f64', pa_int = 2037198692, pa_long_int = 7236283442049920868, pa_long_long_int = 7236283442049920868,
pa_u_int = 2037198692, pa_u_long_int = 7236283442049920868, pa_u_long_long_int = 7236283442049920868, pa_double = 5.6278881776203929e+175,
pa_long_double = <invalid float value>, pa_string = 0x646c7173796d2f64 <error: Cannot access memory at address 0x646c7173796d2f64>,
pa_wstring = 0x646c7173796d2f64 <error: Cannot access memory at address 0x646c7173796d2f64>, pa_pointer = 0x646c7173796d2f64, pa_user = 0x646c7173796d2f64}
space = <optimized out>
is_long_double = <optimized out>
---Type <return> to continue, or q <return> to quit---
is_long = <optimized out>
step3b_jumps = {0 <repeats 11 times>, 1760, 0, 0, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, 0, 0, 0}
step4_jumps = {0 <repeats 14 times>, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, 0, 0, 0}
prec = <optimized out>
_buffer = {__routine = 0xffffffffffffffff, __arg = 0x100000000, __canceltype = 0, __prev = 0x756b43e300000000}
_avail = <optimized out>
thousands_sep = 0x0
grouping = 0xffffffffffffffff <error: Cannot access memory at address 0xffffffffffffffff>
done = 87
f = <optimized out>
lead_str_end = 0x557c27af4386 "%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n"
end_of_spec = <optimized out>
work_buffer = "\260MK\351\374\177\000\000\v\000\000\000>\177\000\000\332u\vF>\177\000\000\000\000\000\000\000\000\000\000h\r\000\000\000\000\000\000\373u\vF>\177\000\000\331u\vF>\177\000\000\v\000\000\000\000\000\000\000\240MK\351\374\177\000\000\200NK\351\374\177\000\000@\037\355F>\177\000\000\060\037\355F>\177\000\000 \037\355F>\177\000\000\020\037\355F>\177\000\000\000\037\355F>\177\000\000\360\036\355F>\177\000\000 \036\355F>\177\000\000\060\036\355F>\177\000\000\320\036\355F>\177\000\000`\036\355F>\177\000\000\360MK\351\374\177\000\000\000\006\332\016\343Cku\320MK\351\374\177\000\000\320MK\351\374\177\000\000\000OK\351\374\177\000\000"...
workstart = <optimized out>
workend = <optimized out>
ap_save = {{gp_offset = 24, fp_offset = 48, overflow_arg_area = 0x7ffce94b52f0, reg_save_area = 0x7ffce94b5200}}
nspecs_done = 3
save_errno = 11
readonly_format = 0
__PRETTY_FUNCTION__ = "_IO_vfprintf_internal"
__result = <optimized out>
#2 0x00007f3e6c3875f8 in ___vfprintf_chk (fp=fp at entry=0x557c28170060, flag=flag at entry=1,
format=format at entry=0x557c27af4378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", ap=ap at entry=0x7ffce94b5188)
at vfprintf_chk.c:33
_IO_acquire_lock_file = 0x557c28170060
done = <optimized out>
#3 0x00007f3e6c36f37c in __GI___vsyslog_chk (ap=0x7ffce94b5188,
fmt=0x557c27af4378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", flag=1, pri=<optimized out>)
at ../misc/syslog.c:222
buf = 0x0
now_tm = {tm_sec = 4, tm_min = 40, tm_hour = 2, tm_mday = 9, tm_mon = 11, tm_year = 121, tm_wday = 4, tm_yday = 342, tm_isdst = 0, tm_gmtoff = 7200,
tm_zone = 0x557c2812da20 "SAST"}
fd = <optimized out>
failbuf = "`\000\027(|U\000\000\001\000\000\000\000\000\000\000\v\000\000\000\000\000\000\000\310u\vF>"
now = 1639010404
f = 0x557c28170060
---Type <return> to continue, or q <return> to quit---
bufsize = 0
msgoff = 21
saved_errno = 11
clarg = <optimized out>
now_tm = <optimized out>
now = <optimized out>
fd = <optimized out>
f = <optimized out>
buf = <optimized out>
bufsize = <optimized out>
msgoff = <optimized out>
saved_errno = <optimized out>
failbuf = <optimized out>
clarg = <optimized out>
numbuf = <optimized out>
nump = <optimized out>
endp = <optimized out>
pid = <optimized out>
__result = <optimized out>
iov = <optimized out>
v = <optimized out>
__clframe = <optimized out>
__clframe = <optimized out>
ignore1 = <optimized out>
ignore2 = <optimized out>
ignore3 = <optimized out>
ignore = <optimized out>
#4 __syslog_chk (pri=<optimized out>, flag=1, fmt=0x557c27af4378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n")
at ../misc/syslog.c:129
ap = {{gp_offset = 48, fp_offset = 48, overflow_arg_area = 0x7ffce94b52f8, reg_save_area = 0x7ffce94b5200}}
#5 0x0000557c279fa286 in fm_free ()
No symbol table info available.
#6 0x00007f3e45f62994 in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#7 0x00007f3e46001799 in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#8 0x00007f3e460017e9 in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#9 0x00007f3e45fb1263 in wolfSSL_Cleanup () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
---Type <return> to continue, or q <return> to quit---
No symbol table info available.
#10 0x00007f3e45f625af in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#11 0x0000557c279c7c78 in ?? ()
No symbol table info available.
#12 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#13 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#14 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#15 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#16 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#17 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#18 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#19 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#20 0x0000557c279ca654 in destroy_modules ()
No symbol table info available.
#21 0x0000557c279ea2e0 in cleanup ()
No symbol table info available.
#22 0x0000557c279ead2b in shutdown_opensips ()
No symbol table info available.
#23 0x0000557c279c68e7 in handle_sigs ()
No symbol table info available.
#24 0x0000557c27922343 in main ()
No symbol table info available.
AND:
#0 __strlen_sse2 () at ../sysdeps/x86_64/multiarch/../strlen.S:120
No locals.
#1 0x00007f5ac10265d3 in _IO_vfprintf_internal (s=s at entry=0x55fdfaba8060,
format=format at entry=0x55fdf9265378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", ap=0x7fffffd36458)
at vfprintf.c:1643
len = <optimized out>
string_malloced = 0
string = 0xfffcd1850f023b48 <error: Cannot access memory at address 0xfffcd1850f023b48>
width = <optimized out>
signed_number = <optimized out>
is_short = 0
spec = 115 's'
ptr = <optimized out>
ptr = <optimized out>
outc = <optimized out>
step0_jumps = {0, 160, 64, 1664, 1576, 1464, 1360, 2216, 2912, 208, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256,
2624, -1344, -1344, 2536}
base = <optimized out>
pad = <optimized out>
offset = <optimized out>
offset = <optimized out>
step1_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, -1344, -1344, 0}
args_value = <optimized out>
is_negative = <optimized out>
number = <optimized out>
use_outdigits = <optimized out>
step2_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, -1344, -1344, 0}
alt = 0
left = 0
showsign = <optimized out>
group = 0
is_char = 0
step3a_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 0, 0, 0, -760, 3024, 3000, 3104, 2128, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0}
the_arg = {pa_wchar = 0 L'\000', pa_int = 0, pa_long_int = 0, pa_long_long_int = 0, pa_u_int = 0, pa_u_long_int = 0, pa_u_long_long_int = 0, pa_double = 0,
pa_long_double = <invalid float value>, pa_string = 0x0, pa_wstring = 0x0, pa_pointer = 0x0, pa_user = 0x0}
space = <optimized out>
is_long_double = <optimized out>
is_long = <optimized out>
step3b_jumps = {0 <repeats 11 times>, 1760, 0, 0, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, 0, 0, 0}
---Type <return> to continue, or q <return> to quit---
step4_jumps = {0 <repeats 14 times>, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, 0, 0, 0}
prec = <optimized out>
_buffer = {__routine = 0x14, __arg = 0x8b6761cc37031500, __canceltype = -1060996992, __prev = 0x7f5ac0bd4020}
_avail = <optimized out>
thousands_sep = 0x0
grouping = 0xffffffffffffffff <error: Cannot access memory at address 0xffffffffffffffff>
done = 87
f = <optimized out>
lead_str_end = 0x55fdf9265386 "%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n"
end_of_spec = <optimized out>
work_buffer = "\036\377,\347ir\231crxiz\v\327Q_\000\000\000\000\000\000\000\000\000\025\003\067\314ag\213\243B\324P\213j\301\256\003\000\000\000\000\000\000\000\070A\275\300Z\177\000\000\000\000\000\000\000\000\000\000\024\000\000\000\000\000\000\000\273\343%\371\375U\000\000\000п\300Z\177\000\000Q\215\023\371\375U\000\000p`\323\377\377\177\000\000\340\000\000\000\000\000\000\000@\000\000\000\017\000\000\000\a\000\000\000\000\000\000\000\360\000\000\000\000\000\000\000x\260\262\372\375U\000\000\020\001\000\000\000\000\000\000\260\377\377\377\377\377\377\377\003\000\000\000\000\000\000\000\000\025\003\067\314ag\213\240`\323\377\377\177\000\000\240`\323\377\377\177\000\000\320a\323\377\377\177\000\000\225"...
workstart = <optimized out>
workend = <optimized out>
ap_save = {{gp_offset = 24, fp_offset = 48, overflow_arg_area = 0x7fffffd365c0, reg_save_area = 0x7fffffd364d0}}
nspecs_done = 3
save_errno = 11
readonly_format = 0
__PRETTY_FUNCTION__ = "_IO_vfprintf_internal"
__result = <optimized out>
#2 0x00007f5ac10fb5f8 in ___vfprintf_chk (fp=fp at entry=0x55fdfaba8060, flag=flag at entry=1,
format=format at entry=0x55fdf9265378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", ap=ap at entry=0x7fffffd36458)
at vfprintf_chk.c:33
_IO_acquire_lock_file = 0x55fdfaba8060
done = <optimized out>
#3 0x00007f5ac10e337c in __GI___vsyslog_chk (ap=0x7fffffd36458,
fmt=0x55fdf9265378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", flag=1, pri=<optimized out>)
at ../misc/syslog.c:222
buf = 0x0
now_tm = {tm_sec = 37, tm_min = 39, tm_hour = 2, tm_mday = 9, tm_mon = 11, tm_year = 121, tm_wday = 4, tm_yday = 342, tm_isdst = 0, tm_gmtoff = 7200,
tm_zone = 0x55fdfab65a20 "SAST"}
fd = <optimized out>
failbuf = "0\000\000\000\060\000\000\000\200e\323\377\377\177\000\000\300d\323\377\377\177\000\000\000\025\003", <incomplete sequence \314>
now = 1639010377
f = 0x55fdfaba8060
bufsize = 0
---Type <return> to continue, or q <return> to quit---
msgoff = 21
saved_errno = 11
clarg = <optimized out>
now_tm = <optimized out>
now = <optimized out>
fd = <optimized out>
f = <optimized out>
buf = <optimized out>
bufsize = <optimized out>
msgoff = <optimized out>
saved_errno = <optimized out>
failbuf = <optimized out>
clarg = <optimized out>
numbuf = <optimized out>
nump = <optimized out>
endp = <optimized out>
pid = <optimized out>
__result = <optimized out>
iov = <optimized out>
v = <optimized out>
__clframe = <optimized out>
__clframe = <optimized out>
ignore1 = <optimized out>
ignore2 = <optimized out>
ignore3 = <optimized out>
ignore = <optimized out>
#4 __syslog_chk (pri=<optimized out>, flag=1, fmt=0x55fdf9265378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n")
at ../misc/syslog.c:129
ap = {{gp_offset = 48, fp_offset = 48, overflow_arg_area = 0x7fffffd365c8, reg_save_area = 0x7fffffd364d0}}
#5 0x000055fdf916b286 in fm_free ()
No symbol table info available.
#6 0x00007f5a9acd6994 in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#7 0x00007f5a9ad75799 in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#8 0x00007f5a9ace6e25 in _wolfssl_tls_read () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#9 0x00007f5a9a87d55f in tls_read () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_mgm.so
No symbol table info available.
---Type <return> to continue, or q <return> to quit---
#10 0x00007f5a9aa8e51e in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/proto_tls.so
No symbol table info available.
#11 0x000055fdf92197fa in ?? ()
No symbol table info available.
#12 0x000055fdf921b042 in tcp_worker_proc_loop ()
No symbol table info available.
#13 0x000055fdf9227046 in tcp_start_processes ()
No symbol table info available.
#14 0x000055fdf9093059 in main ()
No symbol table info available.
And
#0 __GI_raise (sig=sig at entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
set = {__val = {0, 10045105023597352192, 11, 94549321613408, 0,
10045105023597352192, 11, 1, 94549321613408, 1, 11, 94549295125368, 21,
140027762717882,
140737485432176, 1639010387}}
pid = <optimized out>
tid = <optimized out>
ret = <optimized out>
#1 0x00007f5ac1009921 in __GI_abort () at abort.c:79
save_stage = 1
act = {__sigaction_handler = {sa_handler = 0x7fffffd367f0, sa_sigaction
= 0x7fffffd367f0}, sa_mask = {__val = {140737485432576, 140737485432560, 16,
140027221347480, 140027224679288, 140027221347744, 94549293278303,
140737485432544, 140027116336909, 0, 0, 0, 0, 140027224966528, 37504,
94549295118192}},
sa_flags = -115953018, sa_restorer = 0x0}
sigs = {__val = {32, 0 <repeats 15 times>}}
__cnt = <optimized out>
__set = <optimized out>
__cnt = <optimized out>
__set = <optimized out>
#2 0x000055fdf916b25b in fm_free ()
No symbol table info available.
#3 0x00007f5a9acd6994 in ?? () from
/usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#4 0x00007f5a9ad75799 in ?? () from
/usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#5 0x00007f5a9ad757e9 in ?? () from
/usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#6 0x00007f5a9ad25263 in wolfSSL_Cleanup () from
/usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#7 0x00007f5a9acd65af in ?? () from
/usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#8 0x000055fdf9138c78 in ?? ()
No symbol table info available.
#9 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#10 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#11 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#12 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#13 0x000055fdf9138cb1 in ?? ()
---Type <return> to continue, or q <return> to quit---
No symbol table info available.
#14 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#15 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#16 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#17 0x000055fdf913b654 in destroy_modules ()
No symbol table info available.
#18 0x000055fdf915b2e0 in cleanup ()
No symbol table info available.
#19 0x000055fdf915bd2b in shutdown_opensips ()
No symbol table info available.
#20 0x000055fdf9137798 in handle_sigs ()
No symbol table info available.
#21 0x000055fdf9093343 in main ()
No symbol table info available.
Regards
Greg
On 2021-11-18 23:36, Vlad Patrascu wrote:
>
> Hi Gregory,
>
> It is difficult to draw any conclusions without a backtrace so I'm
> hoping you'll be able to get the proper core dumps again in order to
> investigate this further.
>
> Regards,
> --
> Vlad Patrascu
> OpenSIPS Core Developer
> http://www.opensips-solutions.com
> On 12.11.2021 14:44, Gregory Massel wrote:
>>
>> Sorry, I do have the core files, however, my /usr/sbin/opensips is
>> now 3.2.2 and the core dumps were taken against 3.2.3.
>>
>> I should have thought to get the backtrace before downgrading.
>>
>> This is a production server, however, I will try and schedule some
>> downtime and then restore 3.2.3 to generate the backtrace and will
>> try out the nightly release.
>>
>> --Greg
>>
>>
>>
>> _______________________________________________
>> Users mailing list
>> Users at lists.opensips.org
>> http://lists.opensips.org/cgi-bin/mailman/listinfo/users
>
> _______________________________________________
> Users mailing list
> Users at lists.opensips.org
> http://lists.opensips.org/cgi-bin/mailman/listinfo/users
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.opensips.org/pipermail/users/attachments/20211209/17965f7c/attachment-0001.html>
More information about the Users
mailing list