[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.655+0000 D EXECUTOR [NetworkInterfaceASIO] Received remote response: RemoteResponse --  cmd:{ lastErrorObject: { updatedExisting: true, n: 1 }, value: { _id: "ip-10-31-116-34:20004:1447794024:980765113", ping: new Date(1447795494619) }, writeConcernError: { code: 11600, errmsg: "interrupted at shutdown" }, ok: 1.0 }
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.655+0000 D EXECUTOR [ShardWorkFixed-0] Executing a task on behalf of pool ShardWorkFixed
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.655+0000 D EXECUTOR [ShardWorkFixed-0] waiting for work; I am one of 1 thread(s); the minimum number of threads is 1
[ShardedClusterFixture:job0:configsvr:primary] 2015-11-17T21:25:24.655+0000 I COMMAND  [conn21] command config.$cmd command: findAndModify { findAndModify: "lockpings", query: { _id: "ip-10-31-116-34:20004:1447794024:980765113" }, update: { $set: { ping: new Date(1447795524621) } }, upsert: true, writeConcern: { w: "majority", wtimeout: 5000 }, maxTimeMS: 30000 } update: { $set: { ping: new Date(1447795524621) } } ntoreturn:1 ntoskip:0 keyUpdates:1 writeConflicts:0 numYields:0 reslen:349 locks:{ Global: { acquireCount: { r: 2, w: 2 } }, Database: { acquireCount: { w: 2 } }, Collection: { acquireCount: { w: 1 } }, Metadata: { acquireCount: { w: 1 } }, oplog: { acquireCount: { w: 1 } } } protocol:op_command 33ms
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.655+0000 W SHARDING [replSetDistLockPinger] pinging failed for distributed lock pinger :: caused by :: interrupted at shutdown
[ShardedClusterFixture:job0:configsvr:primary] 2015-11-17T21:25:24.656+0000 I REPL     [signalProcessingThread] Stopping replication applier threads
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.712+0000 D EXECUTOR [UserCacheInvalidator] Scheduling remote command request: RemoteCommand -- target:localhost:20000 db:admin expDate:2015-11-17T21:25:54.712+0000 cmd:{ _getUserCacheGeneration: 1, maxTimeMS: 30000 }
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.712+0000 I ASIO     [UserCacheInvalidator] dropping unhealthy pooled connection to localhost:20000
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.713+0000 I -        [NetworkInterfaceASIO] Invariant failure iter != pool.end() src/mongo/executor/connection_pool.cpp 566
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.713+0000 I -        [NetworkInterfaceASIO]
[ShardedClusterFixture:job0:mongos] 
[ShardedClusterFixture:job0:mongos] ***aborting after invariant() failure
[ShardedClusterFixture:job0:mongos] 
[ShardedClusterFixture:job0:mongos] 
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.715+0000 D COMMAND  [PeriodicTaskRunner] task: DBConnectionPool-cleaner took: 0ms
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.715+0000 D -        [PeriodicTaskRunner] cleaning up unused lock buckets of the global lock manager
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.715+0000 D COMMAND  [PeriodicTaskRunner] task: UnusedLockCleaner took: 0ms
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.715+0000 I NETWORK  [PeriodicTaskRunner] Socket closed remotely, no longer connected (idle 9 secs, remote host 127.0.0.1:20001)
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.715+0000 I NETWORK  [PeriodicTaskRunner] Socket closed remotely, no longer connected (idle 9 secs, remote host 127.0.0.1:20002)
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.715+0000 D COMMAND  [PeriodicTaskRunner] task: DBConnectionPool-cleaner took: 0ms
[ShardedClusterFixture:job0:mongos] 2015-11-17T21:25:24.717+0000 F -        [NetworkInterfaceASIO] Got signal: 6 (Aborted).
[ShardedClusterFixture:job0:mongos] 
[ShardedClusterFixture:job0:mongos]  0xc732e2 0xc72229 0xc72a32 0x320160f790 0x3201232625 0x3201233e05 0xbf98cb 0xa0b7a0 0xa0d201 0xa1059c 0xa108d7 0xa11969 0xc8fd31 0xc8ff51 0xc940ef 0xa1aa55 0xe4a7e0 0x3201607a51 0x32012e893d
[ShardedClusterFixture:job0:mongos] ----- BEGIN BACKTRACE -----
[ShardedClusterFixture:job0:mongos] {"backtrace":[{"b":"400000","o":"8732E2"},{"b":"400000","o":"872229"},{"b":"400000","o":"872A32"},{"b":"3201600000","o":"F790"},{"b":"3201200000","o":"32625"},{"b":"3201200000","o":"33E05"},{"b":"400000","o":"7F98CB"},{"b":"400000","o":"60B7A0"},{"b":"400000","o":"60D201"},{"b":"400000","o":"61059C"},{"b":"400000","o":"6108D7"},{"b":"400000","o":"611969"},{"b":"400000","o":"88FD31"},{"b":"400000","o":"88FF51"},{"b":"400000","o":"8940EF"},{"b":"400000","o":"61AA55"},{"b":"400000","o":"A4A7E0"},{"b":"3201600000","o":"7A51"},{"b":"3201200000","o":"E893D"}],"processInfo":{ "mongodbVersion" : "3.2.0-rc3", "gitVersion" : "bd58ea2ba5d17b960981990bb97cab133d7e90ed", "compiledModules" : [ "enterprise" ], "uname" : { "sysname" : "Linux", "release" : "2.6.32-220.el6.x86_64", "version" : "#1 SMP Wed Nov 9 08:03:13 EST 2011", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "7C3205CC858F69DC8E3C6E602ED085F880344116" }, { "b" : "7FFF858CB000", "elfType" : 3, "buildId" : "08F634A1D22DEFF00461D50A7699DACDC97657BF" }, { "path" : "/usr/lib64/libsasl2.so.2", "elfType" : 3, "buildId" : "E0AEE889D5BF1373F2F9EE0D448DBF3F5B5113F0" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "0C72521270790A1BD52C8F6B989EEA5A575085BF" }, { "b" : "7F22BF775000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "93610457BCF424BEBBF1F3FB44E51B51B50F2B55" }, { "b" : "7F22BF392000", "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "06DDBB192AF74F99DB58F2150BFB83F42F5EBAD3" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "58C5A5FF5C82D7BE3113BE36DD87C7004E3C4DB1" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "B5AE05CEDC0CE917F50A3A468CFA2ACD8592E8F6" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "28AF9321EBEA9D172CA43E11A60E02D0F7014870" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "989FE3A42CA8CEBDCC185A743896F23A0CF537ED" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "2AC15B051D1B8B53937E3341EA931D0E96F745D9" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "D467973C46E563CDCF64B5F12B2D6A50C7A25BA1" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "A6D15926E61580E250ED91F84FF7517F3970CD83" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "04202A4A8BE624D2193E812A25589E2DD02D5B5C" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F704FA7D21D05EF31E90FB4890FCA7F3D91DA138" }, { "path" : "/lib64/libcrypt.so.1", "elfType" : 3, "buildId" : "128802B73016BE233837EA9F2DCBC2153ACC2D6A" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "DC11D5D89BDC77FF242481122D51E5A08DB60DA8" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "15782495E3AF093E67DDAE9A86436FFC6B3CC4D3" }, { "b" : "7EF0BC38A000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "13FFCD68952B7715DDF34C9321D82E3041EA9006" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "44A3A1C1891B4C8170C3DB80E7117A022E5EECD0" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "3BCCABE75DC61BBA81AAE45D164E26EF4F9F55DB" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "D053BB4FF0C2FC983842F81598813B9B931AD0D1" }, { "path" : "/lib64/libfreebl3.so", "elfType" : 3, "buildId" : "58BAC04A1DB3964A8F594EFFBE4838AD01214EDC" }, { "b" : "7EF0BC969000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "2D0F26E648D9661ABD83ED8B4BBE8F2CFA50393B" }, { "b" : "7F22BED5E000", "path" : "/usr/lib64/sasl2/libsasldb.so", "elfType" : 3, "buildId" : "4514552B5354286A143770420B38F2D5985D7FA1" }, { "path" : "/lib64/libdb-4.7.so", "elfType" : 3, "buildId" : "437CA0AB593A7383FF1A1700D14AF4998FE93CF3" }, { "b" : "7F22BEB59000", "path" : "/usr/lib64/sasl2/libanonymous.so", "elfType" : 3, "buildId" : "EEAA33A75735D35F4BF25C3C2830B8C90ABDD8B5" }, { "b" : "7F22BE94C000", "path" : "/usr/lib64/sasl2/libdigestmd5.so", "elfType" : 3, "buildId" : "34D8E3E2565DEF4A685D6976831B0372AD456993" }, { "b" : "7F22BE746000", "path" : "/usr/lib64/sasl2/libcrammd5.so", "elfType" : 3, "buildId" : "4CC7E695963F5C8B772EDFF456DB67F89E58FBD6" }, { "b" : "7F22BE541000", "path" : "/usr/lib64/sasl2/libplain.so", "elfType" : 3, "buildId" : "F8DDC7A3CA1CE5B75719AE0DC821647B609D17B6" }, { "b" : "7F22BE33C000", "path" : "/usr/lib64/sasl2/liblogin.so", "elfType" : 3, "buildId" : "9D19F93E342AA4EE2D646E64642625F365056E5C" }, { "b" : "7F22BE134000", "path" : "/usr/lib64/sasl2/libgssapiv2.so", "elfType" : 3, "buildId" : "F7BCE9C6BFF4EAF0CB3142B299CF22D094CE4F04" } ] }}
[ShardedClusterFixture:job0:mongos]  mongos(_ZN5mongo15printStackTraceERSo+0x32) [0xc732e2]
[ShardedClusterFixture:job0:mongos]  mongos(+0x872229) [0xc72229]
[ShardedClusterFixture:job0:mongos]  mongos(+0x872A32) [0xc72a32]
[ShardedClusterFixture:job0:mongos]  libpthread.so.0(+0xF790) [0x320160f790]
[ShardedClusterFixture:job0:mongos]  libc.so.6(gsignal+0x35) [0x3201232625]
[ShardedClusterFixture:job0:mongos]  libc.so.6(abort+0x175) [0x3201233e05]
[ShardedClusterFixture:job0:mongos]  mongos(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0xbf98cb]
[ShardedClusterFixture:job0:mongos]  mongos(_ZN5mongo8executor14ConnectionPool12SpecificPool22takeFromProcessingPoolEPNS1_19ConnectionInterfaceE+0x0) [0xa0b7a0]
[ShardedClusterFixture:job0:mongos]  mongos(+0x60D201) [0xa0d201]
[ShardedClusterFixture:job0:mongos]  mongos(+0x61059C) [0xa1059c]
[ShardedClusterFixture:job0:mongos]  mongos(+0x6108D7) [0xa108d7]
[ShardedClusterFixture:job0:mongos]  mongos(+0x611969) [0xa11969]
[ShardedClusterFixture:job0:mongos]  mongos(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code+0x2F1) [0xc8fd31]
[ShardedClusterFixture:job0:mongos]  mongos(_ZN4asio6detail9scheduler3runERSt10error_code+0xC1) [0xc8ff51]
[ShardedClusterFixture:job0:mongos]  mongos(_ZN4asio10io_service3runEv+0x2F) [0xc940ef]
[ShardedClusterFixture:job0:mongos]  mongos(+0x61AA55) [0xa1aa55]
[ShardedClusterFixture:job0:mongos]  mongos(+0xA4A7E0) [0xe4a7e0]
[ShardedClusterFixture:job0:mongos]  libpthread.so.0(+0x7A51) [0x3201607a51]
[ShardedClusterFixture:job0:mongos]  libc.so.6(clone+0x6D) [0x32012e893d]
[ShardedClusterFixture:job0:mongos] -----  END BACKTRACE  -----