Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-23168

Shell killOperationsOnAllConnections doesn't handle currentOp failure

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Minor - P4 Minor - P4
    • 3.3.11
    • Affects Version/s: None
    • Component/s: Shell
    • Fully Compatible
    • ALL

      When the user interrupts the shell with Ctrl-C (or similar), the shell will check to see if any operations are in progress, and if so, prompt the user to kill them. However, after getting the currentOp results, the code assumes that there will be an inprog field — but this is not present if the currentOp fails (for example, due to a lack of auth). This causes an assertion to be tripped inside BSONElement::embeddedObject().

      void ConnectionRegistry::killOperationsOnAllConnections(bool withPrompt) const {
          Prompter prompter("do you want to kill the current op(s) on the server?");
          stdx::lock_guard<stdx::mutex> lk(_mutex);
          for (map<string, set<string>>::const_iterator i = _connectionUris.begin();
               i != _connectionUris.end();
               ++i) {
              auto status = ConnectionString::parse(i->first);
              if (!status.isOK()) {
                  continue;
              }
      
              const ConnectionString cs(status.getValue());
      
              string errmsg;
              std::unique_ptr<DBClientWithCommands> conn(cs.connect(errmsg));
              if (!conn) {
                  continue;
              }
      
              const set<string>& uris = i->second;
      
              BSONObj currentOpRes;
              conn->runPseudoCommand("admin", "currentOp", "$cmd.sys.inprog", {}, currentOpRes);
              auto inprog = currentOpRes["inprog"].embeddedObject();
              BSONForEach(op, inprog) {
                  if (uris.count(op["client"].String())) {
                      if (!withPrompt || prompter.confirm()) {
                          BSONObjBuilder cmdBob;
                          BSONObj info;
                          cmdBob.append("op", op["opid"]);
                          auto cmdArgs = cmdBob.done();
                          conn->runPseudoCommand("admin", "killOp", "$cmd.sys.killop", cmdArgs, info);
                      } else {
                          return;
                      }
                  }
              }
          }
      }
      

      Repro:

      • Start auth mongod
      • Run a slightly slow non-privileged command
      • Immediately hit Ctrl-C
      $ mlaunch init --binarypath /m/3.2.4/bin --single --auth --port 1111
      launching: /m/3.2.4/bin/mongod on port 1111
      $ /m/3.2.4/bin/mongo --port 1111
      MongoDB shell version: 3.2.4
      connecting to: 127.0.0.1:1111/test
      > db.runCommand({ping:1, payload:(new Array(1048576)).join('x')})
      ^C2016-03-16T19:03:03.458+1100 I -        [main] Assertion failure isABSONObj() src/mongo/bson/bsonelement.cpp 407
      2016-03-16T19:03:03.467+1100 I CONTROL  [main]
       0x961222 0x910248 0x9029c4 0x71b462 0x8eb5d9 0x7002a0 0x7f8c11174c30 0x7f8c11510412 0x1038b1c 0x8bfdeb 0x8c21a0 0x701e7f 0x7064cb 0x7f8c1115fec5 0x6ff859
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"561222","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"510248","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"5029C4","s":"_ZN5mongo12verifyFailedEPKcS1_j"},{"b":"400000","o":"31B462","s":"_ZNK5mongo11BSONElement14embeddedObjectEv"},{"b":"400000","o":"4EB5D9","s":"_ZNK5mongo11shell_utils18ConnectionRegistry30killOperationsOnAllConnectionsEb"},{"b":"400000","o":"3002A0","s":"_ZN5mongo11exitCleanlyENS_8ExitCodeE"},{"b":"7F8C1113E000","o":"36C30"},{"b":"7F8C11504000","o":"C412","s":"pthread_cond_wait"},{"b":"400000","o":"C38B1C","s":"_ZNSt18condition_variable4waitERSt11unique_lockISt5mutexE"},{"b":"400000","o":"4BFDEB","s":"_ZN5mongo5mozjs15MozJSProxyScope15runOnImplThreadESt8functionIFvvEE"},{"b":"400000","o":"4C21A0","s":"_ZN5mongo5mozjs15MozJSProxyScope4execENS_10StringDataERKSsbbbi"},{"b":"400000","o":"301E7F","s":"_Z5_mainiPPcS0_"},{"b":"400000","o":"3064CB","s":"main"},{"b":"7F8C1113E000","o":"21EC5","s":"__libc_start_main"},{"b":"400000","o":"2FF859"}],"processInfo":{ "mongodbVersion" : "3.2.4", "gitVersion" : "e2ee9ffcf9f5a94fad76802e28cc978718bb7a30", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.13.0-39-generic", "version" : "#66-Ubuntu SMP Tue Oct 28 13:30:27 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFFC70FE000", "elfType" : 3 }, { "b" : "7F8C12146000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7F8C11F42000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }, { "b" : "7F8C11C3E000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7F8C11938000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }, { "b" : "7F8C11722000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7F8C11504000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }, { "b" : "7F8C1113E000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }, { "b" : "7F8C1234E000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
       mongo(_ZN5mongo15printStackTraceERSo+0x32) [0x961222]
       mongo(_ZN5mongo10logContextEPKc+0x138) [0x910248]
       mongo(_ZN5mongo12verifyFailedEPKcS1_j+0xB4) [0x9029c4]
       mongo(_ZNK5mongo11BSONElement14embeddedObjectEv+0x32) [0x71b462]
       mongo(_ZNK5mongo11shell_utils18ConnectionRegistry30killOperationsOnAllConnectionsEb+0x3D9) [0x8eb5d9]
       mongo(_ZN5mongo11exitCleanlyENS_8ExitCodeE+0x40) [0x7002a0]
       libc.so.6(+0x36C30) [0x7f8c11174c30]
       libpthread.so.0(pthread_cond_wait+0xC2) [0x7f8c11510412]
       mongo(_ZNSt18condition_variable4waitERSt11unique_lockISt5mutexE+0xC) [0x1038b1c]
       mongo(_ZN5mongo5mozjs15MozJSProxyScope15runOnImplThreadESt8functionIFvvEE+0x10B) [0x8bfdeb]
       mongo(_ZN5mongo5mozjs15MozJSProxyScope4execENS_10StringDataERKSsbbbi+0xB0) [0x8c21a0]
       mongo(_Z5_mainiPPcS0_+0xFFF) [0x701e7f]
       mongo(main+0x2B) [0x7064cb]
       libc.so.6(__libc_start_main+0xF5) [0x7f8c1115fec5]
       mongo(+0x2FF859) [0x6ff859]
      -----  END BACKTRACE  -----
      error:assertion src/mongo/bson/bsonelement.cpp:407
      2016-03-16T19:03:03.497+1100 I -        [main] Invariant failure _state == State::Idle src/mongo/scripting/mozjs/proxyscope.cpp 275
      2016-03-16T19:03:03.497+1100 I -        [main]
      
      ***aborting after invariant() failure
      
      
      2016-03-16T19:03:03.500+1100 F -        [main] Got signal: 6 (Aborted).
      
       0x961222 0x960149 0x960952 0x7f8c11514340 0x7f8c11174bb9 0x7f8c11177fc8 0x901fab 0x8bff51 0x8c1d41 0x7017cf 0x7064cb 0x7f8c1115fec5 0x6ff859
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"561222","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"560149"},{"b":"400000","o":"560952"},{"b":"7F8C11504000","o":"10340"},{"b":"7F8C1113E000","o":"36BB9","s":"gsignal"},{"b":"7F8C1113E000","o":"39FC8","s":"abort"},{"b":"400000","o":"501FAB","s":"_ZN5mongo15invariantFailedEPKcS1_j"},{"b":"400000","o":"4BFF51","s":"_ZN5mongo5mozjs15MozJSProxyScope15runOnImplThreadESt8functionIFvvEE"},{"b":"400000","o":"4C1D41","s":"_ZN5mongo5mozjs15MozJSProxyScope4typeEPKc"},{"b":"400000","o":"3017CF","s":"_Z5_mainiPPcS0_"},{"b":"400000","o":"3064CB","s":"main"},{"b":"7F8C1113E000","o":"21EC5","s":"__libc_start_main"},{"b":"400000","o":"2FF859"}],"processInfo":{ "mongodbVersion" : "3.2.4", "gitVersion" : "e2ee9ffcf9f5a94fad76802e28cc978718bb7a30", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.13.0-39-generic", "version" : "#66-Ubuntu SMP Tue Oct 28 13:30:27 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFFC70FE000", "elfType" : 3 }, { "b" : "7F8C12146000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7F8C11F42000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }, { "b" : "7F8C11C3E000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7F8C11938000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }, { "b" : "7F8C11722000", "path"
      : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7F8C11504000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }, { "b" : "7F8C1113E000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }, { "b" : "7F8C1234E000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
       mongo(_ZN5mongo15printStackTraceERSo+0x32) [0x961222]
       mongo(+0x560149) [0x960149]
       mongo(+0x560952) [0x960952]
       libpthread.so.0(+0x10340) [0x7f8c11514340]
       libc.so.6(gsignal+0x39) [0x7f8c11174bb9]
       libc.so.6(abort+0x148) [0x7f8c11177fc8]
       mongo(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0x901fab]
       mongo(_ZN5mongo5mozjs15MozJSProxyScope15runOnImplThreadESt8functionIFvvEE+0x271) [0x8bff51]
       mongo(_ZN5mongo5mozjs15MozJSProxyScope4typeEPKc+0x71) [0x8c1d41]
       mongo(_Z5_mainiPPcS0_+0x94F) [0x7017cf]
       mongo(main+0x2B) [0x7064cb]
       libc.so.6(__libc_start_main+0xF5) [0x7f8c1115fec5]
       mongo(+0x2FF859) [0x6ff859]
      -----  END BACKTRACE  -----
      Aborted (core dumped)
      

            Assignee:
            nathan.typanski Nathan Typanski
            Reporter:
            kevin.pulo@mongodb.com Kevin Pulo
            Votes:
            0 Vote for this issue
            Watchers:
            8 Start watching this issue

              Created:
              Updated:
              Resolved: