Major - P3 We are running a test server using v3.0.6 with WiredTiger in standalone mode, the docs are like:
{"_id":"Article Title", "status":0}There are about 5 million docs in the collection, and we call findAndModify to get the doc and set the "status" field to 1 every second, from two other test servers.
After running for about 2 hours, the mongod crashed and we got these errors:
2015-09-21T22:51:25.539+0800 I - [conn76] Invariant failure res.existing src/mongo/db/commands/find_and_modify.cpp 492
2015-09-21T22:51:25.545+0800 I CONTROL [conn76]
0xf75549 0xf12271 0xef6022 0x956049 0x956718 0x9cf044 0x9cffcd 0x9d0cdb 0xba223a 0xab3ba0 0x7fb86d 0xf268ab 0x3fe0a07a51 0x3fe06e89ad
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B75549"},{"b":"400000","o":"B12271"},{"b":"400000","o":"AF6022"},{"b":"400000","o":"556049"},{"b":"400000","o":"556718"},{"b":"400000","o":"5CF044"},{"b":"400000","o":"5CFFCD"},{"b":"400000","o":"5D0CDB"},{"b":"400000","o":"7A223A"},{"b":"400000","o":"6B3BA0"},{"b":"400000","o":"3FB86D"},{"b":"400000","o":"B268AB"},{"b":"3FE0A00000","o":"7A51"},{"b":"3FE0600000","o":"E89AD"}],"processInfo":{ "mongodbVersion" : "3.0.6", "gitVersion" : "1ef45a23a4c5e3480ac919b28afcba3c615488f2", "uname" : { "sysname" : "Linux", "release" : "2.6.32-573.3.1.el6.x86_64", "version" : "#1 SMP Thu Aug 13 22:55:16 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "7F7AA372EDE22BA34234ADA10A8AF2E665681140" }, { "b" : "7FFCD0CC2000", "elfType" : 3, "buildId" : "77385010026236DADA9BFEADC4717B7518E8BB29" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "D467973C46E563CDCF64B5F12B2D6A50C7A25BA1" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "6AC11C021C127E84A2724F3A750461A2183B1069" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "78A423B7DD3C00F7A77027D8D764BB432B57F1E7" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "58C5A5FF5C82D7BE3113BE36DD87C7004E3C4DB1" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "B5AE05CEDC0CE917F50A3A468CFA2ACD8592E8F6" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "C03877A9EE01DDC572E2B0F55F64C757773CF8D6" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "989FE3A42CA8CEBDCC185A743896F23A0CF537ED" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9350579A4970FA47F3144AD8F40B183B0954497D" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "285709B5B3FC083FE60B19F5DC2E5282B1708AB1" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "04202A4A8BE624D2193E812A25589E2DD02D5B5C" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "441FA45097A11508E50D55A3D1FF169BF2BE7C62" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "F62622218875795666E08B92D176A50791183EEC" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "B8DEDADC140347276164C729418C7A37B7224135" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "4BDFC7A19C1F328EB4FCFBCE7A1E27606928610D" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F704FA7D21D05EF31E90FB4890FCA7F3D91DA138" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf75549]
mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf12271]
mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xB2) [0xef6022]
mongod(_ZN5mongo16CmdFindAndModify7runImplEPNS_16OperationContextERKSsS4_RKNS_7BSONObjES7_S7_S7_bbbRNS_14BSONObjBuilderERSs+0x1BA9) [0x956049]
mongod(_ZN5mongo16CmdFindAndModify3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x358) [0x956718]
mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9cf044]
mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9cffcd]
mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9d0cdb]
mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xba223a]
mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xab3ba0]
mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x7fb86d]
mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf268ab]
libpthread.so.0(+0x7A51) [0x3fe0a07a51]
libc.so.6(clone+0x6D) [0x3fe06e89ad]
----- END BACKTRACE -----
2015-09-21T22:51:25.545+0800 I - [conn76]
***aborting after invariant() failure