Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-6286

deadlock between db mutex and sm mutex

    • Type: Icon: Bug Bug
    • Resolution: Won't Fix
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: Concurrency
    • None
    • ALL

      This is an old issue, I'm just filing a new ticket as a reminder that we still need to deal with it. The deadlock can also be triggered in a new scenario as a result of some changes I'm making. Specifically, a DocumentSourceMatch $where expression that accesses 'db' could cause a deadlock once we turn off end to end locking in aggregation. A little over a year ago we planned to fix the deadlock by switching to the v8 javascript engine (see SERVER-1762).

      Deadlock occurs when one thread acquires a spider monkey mutex without a db lock (for example to run a reduce function), and then acquires a db lock (for example to access 'db' within that reduce function). The other thread acquires a db lock (for example to run a count) and a spider monkey mutex (to run a $where within that count).

      Test (run each script from a separate client):

      c = db.c;
      c.drop();
      
      c.save( {} );
      c.save( {} );
      
      count = 0;
      while( 1 ) {
          c.mapReduce( function() { emit( 1, { a:1 } ); }, function( k, v ) { return { count:db.c.count() }; }, { out: { inline:1 } } );
          print( ++count );
      }
      
      d = db.d;
      d.drop();
      
      d.save( { a:1 } );
      
      count = 0;
      while( 1 ) {
          d.count( { $where:'this.a==1' } );
          print( ++count );
      }
      

      Stack traces after deadlock:

      #0  0x0000000109892bca in __psynch_cvwait ()
      #1  0x000000010974c274 in _pthread_cond_wait ()
      #2  0x000000010700bcf0 in boost::condition_variable_any::wait<boost::mutex> (this=0x7fdf33405750, m=@0x7fdf33405710) at condition_variable.hpp:137
      #3  0x0000000106f130f5 in mongo::QLock::lock_r (this=0x7fdf33405710) at qlock.h:157
      #4  0x0000000106f15d07 in mongo::WrapperForQLock::lock_r (this=0x7fdf33405710) at d_concurrency.cpp:111
      #5  0x0000000106f0ad17 in mongo::Lock::DBRead::lockTop (this=0x7fdf334990b0, ls=@0x7fdf334142d8) at d_concurrency.cpp:681
      #6  0x0000000106f0bfb1 in mongo::Lock::DBRead::lockDB (this=0x7fdf334990b0, ns=@0x7fdf334990e8) at d_concurrency.cpp:592
      #7  0x0000000106f0dbc7 in mongo::Lock::DBRead::DBRead (this=0x7fdf334990b0, ns=@0x1162f2bc0) at d_concurrency.cpp:609
      #8  0x0000000107093f36 in mongo::Client::ReadContext::ReadContext (this=0x1162f2fd8, ns=@0x1162f2ff0, path=@0x1162f3338, doauth=true) at client.cpp:220
      #9  0x00000001071afb84 in mongo::execCommand (c=0x10862c840, client=@0x7fdf334141a0, queryOptions=0, cmdns=0x7fdf33498964 "test.$cmd", cmdObj=@0x1162f3628, result=@0x1162f3ba8, fromRepl=false) at dbcommands.cpp:1942
      #10 0x00000001071b0d60 in mongo::_runCommands (ns=0x7fdf33498964 "test.$cmd", _cmdobj=@0x1162f3c88, b=@0x1162f3c00, anObjBuilder=@0x1162f3ba8, fromRepl=false, queryOptions=0) at dbcommands.cpp:2016
      #11 0x0000000106cbef25 in mongo::runCommands (ns=0x7fdf33498964 "test.$cmd", jsobj=@0x1162f3c88, curop=@0x7fdf3386b000, b=@0x1162f3c00, anObjBuilder=@0x1162f3ba8, fromRepl=false, queryOptions=0) at query.cpp:43
      #12 0x0000000106cbf9fb in mongo::runQuery (m=@0x1162f4ec8, q=@0x1162f4400, curop=@0x7fdf3386b000, result=@0x7fdf33498ab0) at query.cpp:919
      #13 0x00000001072947a2 in receivedQuery (c=@0x7fdf334141a0, dbresponse=@0x1162f4d98, m=@0x1162f4ec8) at instance.cpp:244
      #14 0x00000001072994d0 in mongo::assembleResponse (m=@0x1162f4ec8, dbresponse=@0x1162f4d98, remote=@0x10862cfb0) at instance.cpp:383
      #15 0x000000010728c919 in mongo::DBDirectClient::call (this=0x7fdf335393b0, toSend=@0x1162f4ec8, response=@0x7fdf33498420, assertOk=false, actualServer=0x7fdf33498358) at instance.cpp:864
      #16 0x000000010728cbbd in non-virtual thunk to mongo::DBDirectClient::call(mongo::Message&, mongo::Message&, bool, std::string*) () at instance.cpp:877
      #17 0x0000000106fe8bae in mongo::DBClientCursor::init (this=0x7fdf33498330) at dbclientcursor.cpp:66
      #18 0x0000000106ba8d1d in mongo::DBClientBase::query (this=0x7fdf335393b0, ns=@0x1162f5348, query=@0x1162f5168, nToReturn=-1, nToSkip=0, fieldsToReturn=0x0, queryOptions=0, batchSize=0) at dbclient.cpp:776
      #19 0x000000010728c222 in mongo::DBDirectClient::query (this=0x7fdf335393b0, ns=@0x1162f5348, query=@0x1162f5388, nToReturn=-1, nToSkip=0, fieldsToReturn=0x0, queryOptions=0, batchSize=0) at instance.cpp:884
      #20 0x000000010734d1fe in mongo::mongo_find (cx=0x7fdf33536040, obj=0x7fdf34046e80, argc=7, argv=0x7fdf3383ef20, rval=0x1162f56b0) at sm_db.cpp:373
      #21 0x0000000106bca3e9 in js_Invoke (cx=0x7fdf33536040, argc=7, flags=0) at jsinterp.c:1375
      #22 0x0000000106bdb514 in js_Interpret (cx=0x7fdf33536040, pc=0x7fdf334656ae ":", result=0x1162f6878) at jsinterp.c:3944
      #23 0x0000000106bca47f in js_Invoke (cx=0x7fdf33536040, argc=2, flags=2) at jsinterp.c:1394
      #24 0x0000000106bca936 in js_InternalInvoke (cx=0x7fdf33536040, obj=0x7fdf34024d90, fval=140596619165568, flags=0, argc=2, argv=0x7fdf334961b0, rval=0x1162f6cb8) at jsinterp.c:1469
      #25 0x0000000106d37d43 in JS_CallFunction (cx=0x7fdf33536040, obj=0x7fdf34024d90, fun=0x7fdf338be530, argc=2, argv=0x7fdf334961b0, rval=0x1162f6cb8) at jsapi.c:4309
      #26 0x0000000107372d87 in mongo::SMScope::invoke (this=0x7fdf3353bf00, func=0x7fdf338be530, args=0x1162f7140, recv=0x0, timeoutMs=0, ignoreReturn=false, readOnlyArgs=true, readOnlyRecv=true) at engine_spidermonkey.cpp:1829
      #27 0x000000010736c745 in mongo::SMScope::invoke (this=0x7fdf3353bf00, funcAddr=140596619240752, args=0x1162f7140, recv=0x0, timeoutMs=0, ignoreReturn=false, readOnlyArgs=true, readOnlyRecv=true) at engine_spidermonkey.cpp:1857
      #28 0x000000010715943e in mongo::PooledScope::invoke (this=0x7fdf334968b0, func=140596619240752, args=0x1162f7140, recv=0x0, timeoutMs=0, ignoreReturn=false, readOnlyArgs=true, readOnlyRecv=true) at engine.cpp:433
      #29 0x000000010728bcce in mongo::Scope::invokeSafe (this=0x7fdf334968b0, func=140596619240752, args=0x1162f7140, recv=0x0, timeoutMs=0, ignoreReturn=false, readOnlyArgs=true, readOnlyRecv=true) at engine.h:89
      #30 0x000000010727a667 in mongo::mr::JSReducer::_reduce (this=0x7fdf33524b40, tuples=@0x7fdf334979f0, key=@0x1162f73d8, endSizeEstimate=@0x1162f73d4) at mr.cpp:191
      #31 0x0000000107267a1a in mongo::mr::JSReducer::reduce (this=0x7fdf33524b40, tuples=@0x7fdf334979f0) at mr.cpp:106
      #32 0x000000010726bc65 in mongo::mr::State::reduceInMemory (this=0x1162f7aa8) at mr.cpp:868
      #33 0x000000010727f4c9 in mongo::mr::MapReduceCommand::run (this=0x10862cf70, dbname=@0x1162f8b68, cmd=@0x1162f8fe8, unnamed_arg=0, errmsg=@0x1162f8640, result=@0x1162f9568, fromRepl=false) at mr.cpp:1163
      #34 0x00000001071adb9e in mongo::_execCommand (c=0x10862cf70, dbname=@0x1162f8b68, cmdObj=@0x1162f8fe8, queryOptions=0, result=@0x1162f9568, fromRepl=false) at dbcommands.cpp:1805
      #35 0x00000001071af90b in mongo::execCommand (c=0x10862cf70, client=@0x7fdf334141a0, queryOptions=0, cmdns=0x7fdf33847414 "test.$cmd", cmdObj=@0x1162f8fe8, result=@0x1162f9568, fromRepl=false) at dbcommands.cpp:1932
      #36 0x00000001071b0d60 in mongo::_runCommands (ns=0x7fdf33847414 "test.$cmd", _cmdobj=@0x1162f9648, b=@0x1162f95c0, anObjBuilder=@0x1162f9568, fromRepl=false, queryOptions=0) at dbcommands.cpp:2016
      #37 0x0000000106cbef25 in mongo::runCommands (ns=0x7fdf33847414 "test.$cmd", jsobj=@0x1162f9648, curop=@0x7fdf33831800, b=@0x1162f95c0, anObjBuilder=@0x1162f9568, fromRepl=false, queryOptions=0) at query.cpp:43
      #38 0x0000000106cbf9fb in mongo::runQuery (m=@0x1162faae0, q=@0x1162f9dc0, curop=@0x7fdf33831800, result=@0x7fdf33496940) at query.cpp:919
      #39 0x00000001072947a2 in receivedQuery (c=@0x7fdf334141a0, dbresponse=@0x1162fa7d8, m=@0x1162faae0) at instance.cpp:244
      #40 0x00000001072994d0 in mongo::assembleResponse (m=@0x1162faae0, dbresponse=@0x1162fa7d8, remote=@0x1162fa808) at instance.cpp:383
      #41 0x0000000106adbf97 in mongo::MyMessageHandler::process (this=0x7fdf33413a40, m=@0x1162faae0, port=0x7fdf33546590, le=0x7fdf33416ee0) at db.cpp:193
      #42 0x0000000106ee23e3 in mongo::pms::threadRun (inPort=0x7fdf33546590) at message_server_port.cpp:86
      #43 0x0000000106ee3b20 in boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> >::operator()<void (*)(mongo::MessagingPort*), boost::_bi::list0> (this=0x7fdf335467d0, f=@0x7fdf335467c8, a=@0x1162fae20, unnamed_arg=0) at bind.hpp:253
      #44 0x0000000106ee3b82 in boost::_bi::bind_t<void, void (*)(mongo::MessagingPort*), boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> > >::operator() (this=0x7fdf335467c8) at bind_template.hpp:20
      #45 0x0000000106ee3bad in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(mongo::MessagingPort*), boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> > > >::run (this=0x7fdf335465e0) at thread.hpp:62
      #46 0x0000000106b691f9 in thread_proxy (param=0x7fdf335465e0) at thread.cpp:121
      #47 0x00000001097488bf in _pthread_start ()
      #48 0x000000010974bb75 in thread_start ()
      
      #0  0x0000000109892bf2 in __psynch_mutexwait ()
      #1  0x00000001097471a1 in pthread_mutex_lock ()
      #2  0x0000000106dd7520 in boost::recursive_mutex::lock (this=0x7fdf3340b3b0) at recursive_mutex.hpp:101
      #3  0x0000000106dd75e0 in boost::unique_lock<boost::recursive_mutex>::lock (this=0x10a272628) at locks.hpp:412
      #4  0x0000000106ccb383 in boost::unique_lock<boost::recursive_mutex>::unique_lock (this=0x10a272628, m_=@0x7fdf3340b3b0) at locks.hpp:290
      #5  0x000000010736c19b in mongo::SMScope::setObject (this=0x7fdf33415d30, field=0x1075d9e54 "obj", obj=@0x10a272cc8, readOnly=true) at engine_spidermonkey.cpp:1610
      #6  0x00000001071591ac in mongo::PooledScope::setObject (this=0x7fdf33523bc0, field=0x1075d9e54 "obj", obj=@0x10a272cc8, readOnly=true) at engine.cpp:404
      #7  0x0000000106e058dd in mongo::Where::exec (this=0x7fdf335259f0, obj=@0x10a272cc8) at matcher.cpp:111
      #8  0x0000000106defc7d in mongo::Matcher::matches (this=0x7fdf33525920, jsobj=@0x10a272cc8, details=0x0) at matcher.cpp:1008
      #9  0x000000010741dbb0 in mongo::CoveredIndexMatcher::matches (this=0x7fdf33525840, key=@0x10a272e08, recLoc=@0x10a272e00, details=0x0, keyUsable=false) at matcher_covered.cpp:90
      #10 0x000000010741df58 in mongo::CoveredIndexMatcher::matchesCurrent (this=0x7fdf33525840, cursor=0x7fdf335246f0, details=0x0) at matcher_covered.cpp:61
      #11 0x0000000106e35d87 in mongo::Cursor::currentMatches (this=0x7fdf335246f0, details=0x0) at cursor.h:193
      #12 0x00000001072c7dc4 in mongo::runCount (ns=0x7fdf33510808 "test.d", cmd=@0x10a273fe8, err=@0x10a2733a0, errCode=@0x10a27339c) at count.cpp:78
      #13 0x00000001071b8f41 in mongo::CmdCount::run (this=0x10862c840, dbname=@0x10a273b68, cmdObj=@0x10a273fe8, unnamed_arg=false, errmsg=@0x10a273640, result=@0x10a274568, unnamed_arg=false) at dbcommands.cpp:762
      #14 0x00000001071adb9e in mongo::_execCommand (c=0x10862c840, dbname=@0x10a273b68, cmdObj=@0x10a273fe8, queryOptions=0, result=@0x10a274568, fromRepl=false) at dbcommands.cpp:1805
      #15 0x00000001071afc45 in mongo::execCommand (c=0x10862c840, client=@0x7fdf33503c50, queryOptions=0, cmdns=0x7fdf3407f814 "test.$cmd", cmdObj=@0x10a273fe8, result=@0x10a274568, fromRepl=false) at dbcommands.cpp:1944
      #16 0x00000001071b0d60 in mongo::_runCommands (ns=0x7fdf3407f814 "test.$cmd", _cmdobj=@0x10a274648, b=@0x10a2745c0, anObjBuilder=@0x10a274568, fromRepl=false, queryOptions=0) at dbcommands.cpp:2016
      #17 0x0000000106cbef25 in mongo::runCommands (ns=0x7fdf3407f814 "test.$cmd", jsobj=@0x10a274648, curop=@0x7fdf3400f400, b=@0x10a2745c0, anObjBuilder=@0x10a274568, fromRepl=false, queryOptions=0) at query.cpp:43
      #18 0x0000000106cbf9fb in mongo::runQuery (m=@0x10a275ae0, q=@0x10a274dc0, curop=@0x7fdf3400f400, result=@0x7fdf335105d0) at query.cpp:919
      #19 0x00000001072947a2 in receivedQuery (c=@0x7fdf33503c50, dbresponse=@0x10a2757d8, m=@0x10a275ae0) at instance.cpp:244
      #20 0x00000001072994d0 in mongo::assembleResponse (m=@0x10a275ae0, dbresponse=@0x10a2757d8, remote=@0x10a275808) at instance.cpp:383
      #21 0x0000000106adbf97 in mongo::MyMessageHandler::process (this=0x7fdf33413a40, m=@0x10a275ae0, port=0x7fdf33414150, le=0x7fdf335038a0) at db.cpp:193
      #22 0x0000000106ee23e3 in mongo::pms::threadRun (inPort=0x7fdf33414150) at message_server_port.cpp:86
      #23 0x0000000106ee3b20 in boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> >::operator()<void (*)(mongo::MessagingPort*), boost::_bi::list0> (this=0x7fdf33414f40, f=@0x7fdf33414f38, a=@0x10a275e20, unnamed_arg=0) at bind.hpp:253
      #24 0x0000000106ee3b82 in boost::_bi::bind_t<void, void (*)(mongo::MessagingPort*), boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> > >::operator() (this=0x7fdf33414f38) at bind_template.hpp:20
      #25 0x0000000106ee3bad in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(mongo::MessagingPort*), boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> > > >::run (this=0x7fdf33414d50) at thread.hpp:62
      #26 0x0000000106b691f9 in thread_proxy (param=0x7fdf33414d50) at thread.cpp:121
      #27 0x00000001097488bf in _pthread_start ()
      #28 0x000000010974bb75 in thread_start ()
      

            Assignee:
            Unassigned Unassigned
            Reporter:
            aaron Aaron Staple
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

              Created:
              Updated:
              Resolved: