-
Type: Bug
-
Resolution: Done
-
Priority: Critical - P2
-
None
-
Affects Version/s: None
-
Component/s: MMAPv1
-
None
-
ALL
Hi,
I have a 3.0.14 mmapv1 server repeatedly crashing with the following stack trace
w: 1 } }, Collection: { acquireCount: { W: 1 } } } 109ms 2016-11-17T13:09:43.840+0000 I STORAGE [conn27] _getOpenFile() invalid file index requested 1832673968 2016-11-17T13:09:43.840+0000 I - [conn27] Invariant failure false src/mongo/db/storage/mmap_v1/mmap_v1_extent_manager.cpp 201 2016-11-17T13:09:43.865+0000 I CONTROL [conn27] 0xfc2852 0xf5d8b9 0xf40c89 0xd9509f 0xd95246 0xd9527f 0xda6516 0xda7523 0xd9c12b 0xd9c2d9 0x9303f1 0x930f9c 0xa6435c 0xa65094 0xc0dd32 0xc0e04b 0xc0e72d 0x9d7499 0x9d8cf5 0x9d9325 0x9 dc00c 0x9fed91 0x9ffd9c 0xa00a6b 0xbdb12b 0xae9aaa 0x823335 0xf72189 0x3766607aa1 0x37662e8aad ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"BC2852","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"B5D8B9","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"B40C89","s":"_ZN5mongo15in variantFailedEPKcS1_j"},{"b":"400000","o":"99509F","s":"_ZNK5mongo19MmapV1ExtentManager12_getOpenFileEi"},{"b":"400000","o":"995246","s":"_ZNK5mongo19MmapV1ExtentManager12_recordForV1E RKNS_7DiskLocE"},{"b":"400000","o":"99527F","s":"_ZNK5mongo19MmapV1ExtentManager11recordForV1ERKNS_7DiskLocE"},{"b":"400000","o":"9A6516","s":"_ZN5mongo19SimpleRecordStoreV125_allocFro mExistingExtentsEPNS_16OperationContextEi"},{"b":"400000","o":"9A7523","s":"_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib"},{"b":"400000","o":"99C12B","s":"_ZN5 mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"99C2D9","s":"_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib"},{"b":"4 00000","o":"5303F1","s":"_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"530F9C","s":"_ZN5mongo10Collection14insertDocumentEPNS_16Ope rationContextERKNS_7BSONObjEb"},{"b":"400000","o":"66435C","s":"_ZN5mongo11UpdateStage8doInsertEv"},{"b":"400000","o":"665094","s":"_ZN5mongo11UpdateStage4workEPm"},{"b":"400000","o":" 80DD32","s":"_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"400000","o":"80E04B","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEP NS_8RecordIdE"},{"b":"400000","o":"80E72D","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"400000","o":"5D7499","s":"_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEP NS_7BSONObjEPPNS_16WriteErrorDetailE"},{"b":"400000","o":"5D8CF5","s":"_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS _19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE"},{"b":"400000","o":"5D9325","s":"_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22Bat chedCommandResponseE"},{"b":"400000","o":"5DC00C","s":"_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5FED91","s":"_ZN5m ongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5FFD9C","s":"_ZN5mongo7Command11execCommandEPNS_16OperationConte xtEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb"},{"b":"400000","o":"600A6B","s":"_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorE EERNS_14BSONObjBuilderEbi"},{"b":"400000","o":"7DB12B","s":"_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_"},{"b":"4000 00","o":"6E9AAA","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"423335","s":"_ZN5mongo16MyMessageHandler 7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE"},{"b":"400000","o":"B72189","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"3766600000","o":"7AA1"},{ "b":"3766200000","o":"E8AAD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.0.14", "gitVersion" : "08352afcca24bfc145240a0fac9d28b978ab77f3", "uname" : { "sysname" : "Linux", "rel ease" : "2.6.32-642.6.2.el6.x86_64", "version" : "#1 SMP Wed Oct 26 06:52:09 UTC 2016", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "C7C98E2BEC3ED8 663669759EECCD26477D4AC85B" }, { "b" : "7FFCB8EFD000", "elfType" : 3, "buildId" : "F19AA6D32E36D9FB2A754E6499A6C4F568C419FB" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "bui ldId" : "B3BD4C596D72FCBE4607C86FEEC14F47B46D0DCC" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "7CE5058BE3F2AE6EBC0DF61E790C3FCED1A34C7B" }, { "path" : "/usr/li b64/libcrypto.so.10", "elfType" : 3, "buildId" : "096F60DC43484ECA364206125CED7D563FF9D027" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "86FE5BC1F46B8F8AA9A7A479FF991 900DB93F720" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "045D39F19533291EBD72D0EE0247F9D49BE2521E" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "B60E F3FCE5D4D2D8BAD2585D5CAAA1167B35DBFD" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9A6E4BDFA184364D81F7DFD789474C3FB8F98A00" }, { "path" : "/lib64/libc.so.6", "elf Type" : 3, "buildId" : "F14F3F5B55FC325FDFA7FB6F22F73CA1C6977379" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "DB04BDF55FE66BADD1081D833C17B4881E0EEDB9" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "9B852585C66329AA02EFB28497E652A40F538E78" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "EF3AACAF D6BF71BB861F194C1559153FB0B020E2" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "DDE6774979156442185836150FC0785170F8001F" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "pat h" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A23DAFBCE170763BF1E836A8B26113F9CD20E0DA" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7 F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "DDF6449707FD4C49DDE32A293EEE9AC218BFC460" }, { "path" : "/lib64/libselinux.so.1", "el fType" : 3, "buildId" : "B4576BE308DDCF7BC31F7304E4734C3D846D0236" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xfc2852] mongod(_ZN5mongo10logContextEPKc+0xE9) [0xf5d8b9] mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xB9) [0xf40c89] mongod(_ZNK5mongo19MmapV1ExtentManager12_getOpenFileEi+0x8F) [0xd9509f] mongod(_ZNK5mongo19MmapV1ExtentManager12_recordForV1ERKNS_7DiskLocE+0x16) [0xd95246] mongod(_ZNK5mongo19MmapV1ExtentManager11recordForV1ERKNS_7DiskLocE+0xF) [0xd9527f] mongod(_ZN5mongo19SimpleRecordStoreV125_allocFromExistingExtentsEPNS_16OperationContextEi+0xE6) [0xda6516] mongod(_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib+0x163) [0xda7523] mongod(_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib+0x5B) [0xd9c12b] mongod(_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib+0x79) [0xd9c2d9] mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x61) [0x9303f1] mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x8C) [0x930f9c] mongod(_ZN5mongo11UpdateStage8doInsertEv+0x17C) [0xa6435c] mongod(_ZN5mongo11UpdateStage4workEPm+0xD4) [0xa65094] mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x142) [0xc0dd32] mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x3B) [0xc0e04b] mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xc0e72d] mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x719) [0x9d7499] mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaIS E_EE+0x245) [0x9d8cf5] mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x465) [0x9d9325] mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x17C) [0x9dc00c] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x31) [0x9fed91] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC6C) [0x9ffd9c] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0xa00a6b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x73B) [0xbdb12b] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xABA) [0xae9aaa] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xF5) [0x823335] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) [0xf72189] libpthread.so.0(+0x7AA1) [0x3766607aa1] libc.so.6(clone+0x6D) [0x37662e8aad] ----- END BACKTRACE ----- 2016-11-17T13:09:43.865+0000 I - [conn27] ***aborting after invariant() failure
If you need any other information to debug this please let me know.