-
Type: Bug
-
Resolution: Won't Fix
-
Priority: Major - P3
-
None
-
Affects Version/s: None
-
Component/s: Concurrency
-
None
-
ALL
This is an old issue, I'm just filing a new ticket as a reminder that we still need to deal with it. The deadlock can also be triggered in a new scenario as a result of some changes I'm making. Specifically, a DocumentSourceMatch $where expression that accesses 'db' could cause a deadlock once we turn off end to end locking in aggregation. A little over a year ago we planned to fix the deadlock by switching to the v8 javascript engine (see SERVER-1762).
Deadlock occurs when one thread acquires a spider monkey mutex without a db lock (for example to run a reduce function), and then acquires a db lock (for example to access 'db' within that reduce function). The other thread acquires a db lock (for example to run a count) and a spider monkey mutex (to run a $where within that count).
Test (run each script from a separate client):
c = db.c; c.drop(); c.save( {} ); c.save( {} ); count = 0; while( 1 ) { c.mapReduce( function() { emit( 1, { a:1 } ); }, function( k, v ) { return { count:db.c.count() }; }, { out: { inline:1 } } ); print( ++count ); }
d = db.d; d.drop(); d.save( { a:1 } ); count = 0; while( 1 ) { d.count( { $where:'this.a==1' } ); print( ++count ); }
Stack traces after deadlock:
#0 0x0000000109892bca in __psynch_cvwait () #1 0x000000010974c274 in _pthread_cond_wait () #2 0x000000010700bcf0 in boost::condition_variable_any::wait<boost::mutex> (this=0x7fdf33405750, m=@0x7fdf33405710) at condition_variable.hpp:137 #3 0x0000000106f130f5 in mongo::QLock::lock_r (this=0x7fdf33405710) at qlock.h:157 #4 0x0000000106f15d07 in mongo::WrapperForQLock::lock_r (this=0x7fdf33405710) at d_concurrency.cpp:111 #5 0x0000000106f0ad17 in mongo::Lock::DBRead::lockTop (this=0x7fdf334990b0, ls=@0x7fdf334142d8) at d_concurrency.cpp:681 #6 0x0000000106f0bfb1 in mongo::Lock::DBRead::lockDB (this=0x7fdf334990b0, ns=@0x7fdf334990e8) at d_concurrency.cpp:592 #7 0x0000000106f0dbc7 in mongo::Lock::DBRead::DBRead (this=0x7fdf334990b0, ns=@0x1162f2bc0) at d_concurrency.cpp:609 #8 0x0000000107093f36 in mongo::Client::ReadContext::ReadContext (this=0x1162f2fd8, ns=@0x1162f2ff0, path=@0x1162f3338, doauth=true) at client.cpp:220 #9 0x00000001071afb84 in mongo::execCommand (c=0x10862c840, client=@0x7fdf334141a0, queryOptions=0, cmdns=0x7fdf33498964 "test.$cmd", cmdObj=@0x1162f3628, result=@0x1162f3ba8, fromRepl=false) at dbcommands.cpp:1942 #10 0x00000001071b0d60 in mongo::_runCommands (ns=0x7fdf33498964 "test.$cmd", _cmdobj=@0x1162f3c88, b=@0x1162f3c00, anObjBuilder=@0x1162f3ba8, fromRepl=false, queryOptions=0) at dbcommands.cpp:2016 #11 0x0000000106cbef25 in mongo::runCommands (ns=0x7fdf33498964 "test.$cmd", jsobj=@0x1162f3c88, curop=@0x7fdf3386b000, b=@0x1162f3c00, anObjBuilder=@0x1162f3ba8, fromRepl=false, queryOptions=0) at query.cpp:43 #12 0x0000000106cbf9fb in mongo::runQuery (m=@0x1162f4ec8, q=@0x1162f4400, curop=@0x7fdf3386b000, result=@0x7fdf33498ab0) at query.cpp:919 #13 0x00000001072947a2 in receivedQuery (c=@0x7fdf334141a0, dbresponse=@0x1162f4d98, m=@0x1162f4ec8) at instance.cpp:244 #14 0x00000001072994d0 in mongo::assembleResponse (m=@0x1162f4ec8, dbresponse=@0x1162f4d98, remote=@0x10862cfb0) at instance.cpp:383 #15 0x000000010728c919 in mongo::DBDirectClient::call (this=0x7fdf335393b0, toSend=@0x1162f4ec8, response=@0x7fdf33498420, assertOk=false, actualServer=0x7fdf33498358) at instance.cpp:864 #16 0x000000010728cbbd in non-virtual thunk to mongo::DBDirectClient::call(mongo::Message&, mongo::Message&, bool, std::string*) () at instance.cpp:877 #17 0x0000000106fe8bae in mongo::DBClientCursor::init (this=0x7fdf33498330) at dbclientcursor.cpp:66 #18 0x0000000106ba8d1d in mongo::DBClientBase::query (this=0x7fdf335393b0, ns=@0x1162f5348, query=@0x1162f5168, nToReturn=-1, nToSkip=0, fieldsToReturn=0x0, queryOptions=0, batchSize=0) at dbclient.cpp:776 #19 0x000000010728c222 in mongo::DBDirectClient::query (this=0x7fdf335393b0, ns=@0x1162f5348, query=@0x1162f5388, nToReturn=-1, nToSkip=0, fieldsToReturn=0x0, queryOptions=0, batchSize=0) at instance.cpp:884 #20 0x000000010734d1fe in mongo::mongo_find (cx=0x7fdf33536040, obj=0x7fdf34046e80, argc=7, argv=0x7fdf3383ef20, rval=0x1162f56b0) at sm_db.cpp:373 #21 0x0000000106bca3e9 in js_Invoke (cx=0x7fdf33536040, argc=7, flags=0) at jsinterp.c:1375 #22 0x0000000106bdb514 in js_Interpret (cx=0x7fdf33536040, pc=0x7fdf334656ae ":", result=0x1162f6878) at jsinterp.c:3944 #23 0x0000000106bca47f in js_Invoke (cx=0x7fdf33536040, argc=2, flags=2) at jsinterp.c:1394 #24 0x0000000106bca936 in js_InternalInvoke (cx=0x7fdf33536040, obj=0x7fdf34024d90, fval=140596619165568, flags=0, argc=2, argv=0x7fdf334961b0, rval=0x1162f6cb8) at jsinterp.c:1469 #25 0x0000000106d37d43 in JS_CallFunction (cx=0x7fdf33536040, obj=0x7fdf34024d90, fun=0x7fdf338be530, argc=2, argv=0x7fdf334961b0, rval=0x1162f6cb8) at jsapi.c:4309 #26 0x0000000107372d87 in mongo::SMScope::invoke (this=0x7fdf3353bf00, func=0x7fdf338be530, args=0x1162f7140, recv=0x0, timeoutMs=0, ignoreReturn=false, readOnlyArgs=true, readOnlyRecv=true) at engine_spidermonkey.cpp:1829 #27 0x000000010736c745 in mongo::SMScope::invoke (this=0x7fdf3353bf00, funcAddr=140596619240752, args=0x1162f7140, recv=0x0, timeoutMs=0, ignoreReturn=false, readOnlyArgs=true, readOnlyRecv=true) at engine_spidermonkey.cpp:1857 #28 0x000000010715943e in mongo::PooledScope::invoke (this=0x7fdf334968b0, func=140596619240752, args=0x1162f7140, recv=0x0, timeoutMs=0, ignoreReturn=false, readOnlyArgs=true, readOnlyRecv=true) at engine.cpp:433 #29 0x000000010728bcce in mongo::Scope::invokeSafe (this=0x7fdf334968b0, func=140596619240752, args=0x1162f7140, recv=0x0, timeoutMs=0, ignoreReturn=false, readOnlyArgs=true, readOnlyRecv=true) at engine.h:89 #30 0x000000010727a667 in mongo::mr::JSReducer::_reduce (this=0x7fdf33524b40, tuples=@0x7fdf334979f0, key=@0x1162f73d8, endSizeEstimate=@0x1162f73d4) at mr.cpp:191 #31 0x0000000107267a1a in mongo::mr::JSReducer::reduce (this=0x7fdf33524b40, tuples=@0x7fdf334979f0) at mr.cpp:106 #32 0x000000010726bc65 in mongo::mr::State::reduceInMemory (this=0x1162f7aa8) at mr.cpp:868 #33 0x000000010727f4c9 in mongo::mr::MapReduceCommand::run (this=0x10862cf70, dbname=@0x1162f8b68, cmd=@0x1162f8fe8, unnamed_arg=0, errmsg=@0x1162f8640, result=@0x1162f9568, fromRepl=false) at mr.cpp:1163 #34 0x00000001071adb9e in mongo::_execCommand (c=0x10862cf70, dbname=@0x1162f8b68, cmdObj=@0x1162f8fe8, queryOptions=0, result=@0x1162f9568, fromRepl=false) at dbcommands.cpp:1805 #35 0x00000001071af90b in mongo::execCommand (c=0x10862cf70, client=@0x7fdf334141a0, queryOptions=0, cmdns=0x7fdf33847414 "test.$cmd", cmdObj=@0x1162f8fe8, result=@0x1162f9568, fromRepl=false) at dbcommands.cpp:1932 #36 0x00000001071b0d60 in mongo::_runCommands (ns=0x7fdf33847414 "test.$cmd", _cmdobj=@0x1162f9648, b=@0x1162f95c0, anObjBuilder=@0x1162f9568, fromRepl=false, queryOptions=0) at dbcommands.cpp:2016 #37 0x0000000106cbef25 in mongo::runCommands (ns=0x7fdf33847414 "test.$cmd", jsobj=@0x1162f9648, curop=@0x7fdf33831800, b=@0x1162f95c0, anObjBuilder=@0x1162f9568, fromRepl=false, queryOptions=0) at query.cpp:43 #38 0x0000000106cbf9fb in mongo::runQuery (m=@0x1162faae0, q=@0x1162f9dc0, curop=@0x7fdf33831800, result=@0x7fdf33496940) at query.cpp:919 #39 0x00000001072947a2 in receivedQuery (c=@0x7fdf334141a0, dbresponse=@0x1162fa7d8, m=@0x1162faae0) at instance.cpp:244 #40 0x00000001072994d0 in mongo::assembleResponse (m=@0x1162faae0, dbresponse=@0x1162fa7d8, remote=@0x1162fa808) at instance.cpp:383 #41 0x0000000106adbf97 in mongo::MyMessageHandler::process (this=0x7fdf33413a40, m=@0x1162faae0, port=0x7fdf33546590, le=0x7fdf33416ee0) at db.cpp:193 #42 0x0000000106ee23e3 in mongo::pms::threadRun (inPort=0x7fdf33546590) at message_server_port.cpp:86 #43 0x0000000106ee3b20 in boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> >::operator()<void (*)(mongo::MessagingPort*), boost::_bi::list0> (this=0x7fdf335467d0, f=@0x7fdf335467c8, a=@0x1162fae20, unnamed_arg=0) at bind.hpp:253 #44 0x0000000106ee3b82 in boost::_bi::bind_t<void, void (*)(mongo::MessagingPort*), boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> > >::operator() (this=0x7fdf335467c8) at bind_template.hpp:20 #45 0x0000000106ee3bad in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(mongo::MessagingPort*), boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> > > >::run (this=0x7fdf335465e0) at thread.hpp:62 #46 0x0000000106b691f9 in thread_proxy (param=0x7fdf335465e0) at thread.cpp:121 #47 0x00000001097488bf in _pthread_start () #48 0x000000010974bb75 in thread_start ()
#0 0x0000000109892bf2 in __psynch_mutexwait () #1 0x00000001097471a1 in pthread_mutex_lock () #2 0x0000000106dd7520 in boost::recursive_mutex::lock (this=0x7fdf3340b3b0) at recursive_mutex.hpp:101 #3 0x0000000106dd75e0 in boost::unique_lock<boost::recursive_mutex>::lock (this=0x10a272628) at locks.hpp:412 #4 0x0000000106ccb383 in boost::unique_lock<boost::recursive_mutex>::unique_lock (this=0x10a272628, m_=@0x7fdf3340b3b0) at locks.hpp:290 #5 0x000000010736c19b in mongo::SMScope::setObject (this=0x7fdf33415d30, field=0x1075d9e54 "obj", obj=@0x10a272cc8, readOnly=true) at engine_spidermonkey.cpp:1610 #6 0x00000001071591ac in mongo::PooledScope::setObject (this=0x7fdf33523bc0, field=0x1075d9e54 "obj", obj=@0x10a272cc8, readOnly=true) at engine.cpp:404 #7 0x0000000106e058dd in mongo::Where::exec (this=0x7fdf335259f0, obj=@0x10a272cc8) at matcher.cpp:111 #8 0x0000000106defc7d in mongo::Matcher::matches (this=0x7fdf33525920, jsobj=@0x10a272cc8, details=0x0) at matcher.cpp:1008 #9 0x000000010741dbb0 in mongo::CoveredIndexMatcher::matches (this=0x7fdf33525840, key=@0x10a272e08, recLoc=@0x10a272e00, details=0x0, keyUsable=false) at matcher_covered.cpp:90 #10 0x000000010741df58 in mongo::CoveredIndexMatcher::matchesCurrent (this=0x7fdf33525840, cursor=0x7fdf335246f0, details=0x0) at matcher_covered.cpp:61 #11 0x0000000106e35d87 in mongo::Cursor::currentMatches (this=0x7fdf335246f0, details=0x0) at cursor.h:193 #12 0x00000001072c7dc4 in mongo::runCount (ns=0x7fdf33510808 "test.d", cmd=@0x10a273fe8, err=@0x10a2733a0, errCode=@0x10a27339c) at count.cpp:78 #13 0x00000001071b8f41 in mongo::CmdCount::run (this=0x10862c840, dbname=@0x10a273b68, cmdObj=@0x10a273fe8, unnamed_arg=false, errmsg=@0x10a273640, result=@0x10a274568, unnamed_arg=false) at dbcommands.cpp:762 #14 0x00000001071adb9e in mongo::_execCommand (c=0x10862c840, dbname=@0x10a273b68, cmdObj=@0x10a273fe8, queryOptions=0, result=@0x10a274568, fromRepl=false) at dbcommands.cpp:1805 #15 0x00000001071afc45 in mongo::execCommand (c=0x10862c840, client=@0x7fdf33503c50, queryOptions=0, cmdns=0x7fdf3407f814 "test.$cmd", cmdObj=@0x10a273fe8, result=@0x10a274568, fromRepl=false) at dbcommands.cpp:1944 #16 0x00000001071b0d60 in mongo::_runCommands (ns=0x7fdf3407f814 "test.$cmd", _cmdobj=@0x10a274648, b=@0x10a2745c0, anObjBuilder=@0x10a274568, fromRepl=false, queryOptions=0) at dbcommands.cpp:2016 #17 0x0000000106cbef25 in mongo::runCommands (ns=0x7fdf3407f814 "test.$cmd", jsobj=@0x10a274648, curop=@0x7fdf3400f400, b=@0x10a2745c0, anObjBuilder=@0x10a274568, fromRepl=false, queryOptions=0) at query.cpp:43 #18 0x0000000106cbf9fb in mongo::runQuery (m=@0x10a275ae0, q=@0x10a274dc0, curop=@0x7fdf3400f400, result=@0x7fdf335105d0) at query.cpp:919 #19 0x00000001072947a2 in receivedQuery (c=@0x7fdf33503c50, dbresponse=@0x10a2757d8, m=@0x10a275ae0) at instance.cpp:244 #20 0x00000001072994d0 in mongo::assembleResponse (m=@0x10a275ae0, dbresponse=@0x10a2757d8, remote=@0x10a275808) at instance.cpp:383 #21 0x0000000106adbf97 in mongo::MyMessageHandler::process (this=0x7fdf33413a40, m=@0x10a275ae0, port=0x7fdf33414150, le=0x7fdf335038a0) at db.cpp:193 #22 0x0000000106ee23e3 in mongo::pms::threadRun (inPort=0x7fdf33414150) at message_server_port.cpp:86 #23 0x0000000106ee3b20 in boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> >::operator()<void (*)(mongo::MessagingPort*), boost::_bi::list0> (this=0x7fdf33414f40, f=@0x7fdf33414f38, a=@0x10a275e20, unnamed_arg=0) at bind.hpp:253 #24 0x0000000106ee3b82 in boost::_bi::bind_t<void, void (*)(mongo::MessagingPort*), boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> > >::operator() (this=0x7fdf33414f38) at bind_template.hpp:20 #25 0x0000000106ee3bad in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(mongo::MessagingPort*), boost::_bi::list1<boost::_bi::value<mongo::MessagingPort*> > > >::run (this=0x7fdf33414d50) at thread.hpp:62 #26 0x0000000106b691f9 in thread_proxy (param=0x7fdf33414d50) at thread.cpp:121 #27 0x00000001097488bf in _pthread_start () #28 0x000000010974bb75 in thread_start ()
- related to
-
SERVER-6123 aggregation holds read lock indefinitely without yielding after initial processing of first sort or group stage
- Closed
-
SERVER-2407 Switch to v8
- Closed