ceph read call stacks

jewel-10.2.5

1. 第一步: messenger thread读取message,送入队列

Thread 66 "ms_pipe_read" hit Breakpoint 9, OSD::handle_op (this=this@entry=0x55555ff88000, 
    op=std::shared_ptr (count 2, weak 0) 0x55555fdaa300, osdmap=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr (count 135, weak 1) 0x55556117d180) at osd/OSD.cc:8463
8463	{
(gdb) 
(gdb) 
(gdb) 
(gdb) bt
#0  OSD::handle_op (this=this@entry=0x55555ff88000, op=std::shared_ptr (count 2, weak 0) 0x55555fdaa300, osdmap=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'

std::shared_ptr (count 135, weak 1) 0x55556117d180) at osd/OSD.cc:8463
#1  0x00005555558e16ee in OSD::dispatch_op_fast (this=this@entry=0x55555ff88000, op=std::shared_ptr (count 2, weak 0) 0x55555fdaa300, osdmap=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'

std::shared_ptr (count 135, weak 1) 0x55556117d180) at osd/OSD.cc:6135
#2  0x00005555558e17cf in OSD::dispatch_session_waiting (this=this@entry=0x55555ff88000, session=session@entry=0x55555fdbf400, osdmap=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'

std::shared_ptr (count 135, weak 1) 0x55556117d180) at osd/OSD.cc:5771
#3  0x00005555558e1b75 in OSD::ms_fast_dispatch (this=0x55555ff88000, m=<optimized out>) at osd/OSD.cc:5883
#4  0x000055555608c176 in Messenger::ms_fast_dispatch (m=0x55555ffea680, this=0x55555fe76800) at ./msg/Messenger.h:555
#5  DispatchQueue::fast_dispatch (this=0x55555fe769f8, m=0x55555ffea680) at msg/simple/DispatchQueue.cc:71
#6  0x00005555560d39cb in Pipe::reader (this=0x5555613dc000) at msg/simple/Pipe.cc:1668
#7  0x00005555560dbc1d in Pipe::Reader::entry (this=<optimized out>) at msg/simple/Pipe.h:52
#8  0x00007ffff6e9670a in start_thread (arg=0x7fffd14b9700) at pthread_create.c:333
#9  0x00007ffff4f0f82d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
(gdb) 

2. 第二步, ”tp_osd_tp” thread取出消息,进行read

Thread 48 "tp_osd_tp" hit Breakpoint 8, ReplicatedPG::do_request (this=0x5555603fd000, op=std::shared_ptr (count 3, weak 0) 0x55555fdaaf80, 
    handle=...) at osd/ReplicatedPG.cc:1446
1446	{
(gdb) bt
#0  ReplicatedPG::do_request (this=0x5555603fd000, op=std::shared_ptr (count 3, weak 0) 0x55555fdaaf80, handle=...)
    at osd/ReplicatedPG.cc:1446
#1  0x00005555558d7401 in OSD::dequeue_op (this=0x55555ff88000, pg=..., op=std::shared_ptr (count 3, weak 0) 0x55555fdaaf80, handle=...)
    at osd/OSD.cc:8815
#2  0x00005555558d7652 in PGQueueable::RunVis::operator() (this=this@entry=0x7fffd95896a0, 
    op=std::shared_ptr (count 3, weak 0) 0x55555fdaaf80) at osd/OSD.cc:163
#3  0x00005555558f53f1 in boost::detail::variant::invoke_visitor<PGQueueable::RunVis>::internal_visit<std::shared_ptr<OpRequest> > (
    operand=..., this=<synthetic pointer>) at /usr/include/boost/variant/variant.hpp:978
#4  boost::detail::variant::visitation_impl_invoke_impl<boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*, std::shared_ptr<OpRequest> > (storage=0x7fffd95897e0, visitor=<synthetic pointer>) at /usr/include/boost/variant/detail/visitation_impl.hpp:114
#5  boost::detail::variant::visitation_impl_invoke<boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*, std::shared_ptr<OpRequest>, boost::variant<std::shared_ptr<OpRequest>, PGSnapTrim, PGScrub>::has_fallback_type_> (t=0x0, storage=0x7fffd95897e0, 
    visitor=<synthetic pointer>, internal_which=<optimized out>) at /usr/include/boost/variant/detail/visitation_impl.hpp:157
#6  boost::detail::variant::visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<3l>, std::shared_ptr<OpRequest>, boost::mpl::l_item<mpl_::long_<2l>, PGSnapTrim, boost::mpl::l_item<mpl_::long_<1l>, PGScrub, boost::mpl::l_end> > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*, boost::variant<std::shared_ptr<OpRequest>, PGSnapTrim, PGScrub>::has_fallback_type_> (no_backup_flag=..., storage=0x7fffd95897e0, 
    visitor=<synthetic pointer>, logical_which=<optimized out>, internal_which=<optimized out>)
    at /usr/include/boost/variant/detail/visitation_impl.hpp:238
#7  boost::variant<std::shared_ptr<OpRequest>, PGSnapTrim, PGScrub>::internal_apply_visitor_impl<boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*> (storage=0x7fffd95897e0, visitor=<synthetic pointer>, logical_which=<optimized out>, internal_which=<optimized out>)
    at /usr/include/boost/variant/variant.hpp:2323
#8  boost::variant<std::shared_ptr<OpRequest>, PGSnapTrim, PGScrub>::internal_apply_visitor<boost::detail::variant::invoke_visitor<PGQueueable::RunVis> > (visitor=<synthetic pointer>, this=0x7fffd95897d8) at /usr/include/boost/variant/variant.hpp:2334
#9  boost::variant<std::shared_ptr<OpRequest>, PGSnapTrim, PGScrub>::apply_visitor<PGQueueable::RunVis> (visitor=..., this=0x7fffd95897d8)
    at /usr/include/boost/variant/variant.hpp:2357
#10 boost::apply_visitor<PGQueueable::RunVis, boost::variant<std::shared_ptr<OpRequest>, PGSnapTrim, PGScrub> > (visitable=..., visitor=...)
    at /usr/include/boost/variant/detail/apply_visitor_unary.hpp:70
#11 PGQueueable::run (handle=..., pg=..., osd=<optimized out>, this=0x7fffd95897d8) at osd/OSD.h:392
#12 OSD::ShardedOpWQ::_process (this=0x55555ff89558, thread_index=<optimized out>, hb=<optimized out>) at osd/OSD.cc:8696
#13 0x0000555555fb74ea in ShardedThreadPool::shardedthreadpool_worker (this=0x55555ff88860, thread_index=3) at common/WorkQueue.cc:340
#14 0x0000555555fba600 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#15 0x00007ffff6e9670a in start_thread (arg=0x7fffd958b700) at pthread_create.c:333
#16 0x00007ffff4f0f82d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
(gdb) 

 

Leave a Reply