Hi,
v2.8.0-rc5 crashes during a large data import to a cluster of 3 mongod instances with the error message:
"write results unavailable from xxx.xxx.xx:27011 :: caused by :: Location 17255 error receiving write command response, possible socket exception
The batch insert is failing on the client side but the server log is also crashing.
This occurs when there are 4 parallel imports being run on each of the 3 mongos instances.
mongod log error:
2015-01-28T08:40:01.421+0100 I WRITE [conn78] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c89242228b39f12990b57c", y: 1, o: 10, t: 1395314451000, p: { g: [ 5.9130544653739, 51.650652567424 ], h: 154.0, z: 13.0, v: 3916, q: 2, a: 0 }, r: "B61FA96E-9DB0-4EE2-A40C-858E6C8013A4" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 850ms 2015-01-28T08:40:01.432+0100 I SHARDING [conn70] moveChunk data transfer progress: { active: true, ns: "COLLECTION-tigercluster.COLLECTION", from: "tigersrd3_1/xxx.xxx.xx:270xx", min: { r: "A50BCF9C-EFDF-40FF-B47C-42C728D05762" }, max: { r: "A52F12EE-C721-487F-BABF-914A70930B66" }, shardKeyPattern: { r: 1.0 }, state: "clone", counts: { cloned: 85649, clonedBytes: 16187661, catchup: 0, steady: 0 }, ok: 1.0 } my mem used: 0 2015-01-28T08:40:01.434+0100 I - [conn76] Invariant failure: ret resulted in status UnknownError 2: No such file or directory at src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp 438 2015-01-28T08:40:01.432+0100 I WRITE [conn68] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c89242a3b8d4021e42387d", y: 1, o: 10, t: 1393952186000, p: { g: [ 5.4865192694804, 52.164699210253 ], h: 97.0, z: 7.0, v: 3277, q: 2, a: 0 }, r: "D857EB1F-E235-4C58-9283-AA8C4013C003" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 878ms 2015-01-28T08:40:01.440+0100 I WRITE [conn66] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c8923fcdb479b37f68d14e", y: 1, o: 10, t: 1393635113000, p: { g: [ 5.047504050466, 52.070182255382 ], h: 104.0, z: -4.0, v: 2722, q: 2, a: 0 }, r: "CD9A2E28-DA7C-4191-8B3F-51EEAFF7919F" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 875ms 2015-01-28T08:40:01.440+0100 I WRITE [conn63] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c89240973f4164a0cff8da", y: 1, o: 10, t: 1394193599000, p: { g: [ 4.8112359278986, 52.316372955215 ], h: 328.0, z: 2.0, v: 2805, q: 2, a: 0 }, r: "B58B1CF8-3F50-40F1-846A-C97AF0957F45" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 885ms 2015-01-28T08:40:01.441+0100 I WRITE [conn82] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c892422e2461dc2ba41b92", y: 1, o: 10, t: 1393398599000, p: { g: [ 5.1026667810514, 51.974993455037 ], h: 129.0, z: 3.0, v: 3388, q: 2, a: 0 }, r: "F1BDE97C-5018-45F3-97B0-B61BF1CF1512" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 880ms 2015-01-28T08:40:01.446+0100 I WRITE [conn87] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c892421ccd143413ee1d5f", y: 1, o: 10, t: 1394605462000, p: { g: [ 5.0560193578298, 52.108511417551 ], h: 314.0, z: -2.0, v: 2722, q: 2, a: 0 }, r: "D638759B-3A7A-43EB-83F9-26F8C14E60F6" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 879ms 2015-01-28T08:40:01.457+0100 I CONTROL [conn76] 0xf25749 0xecf571 0xeb629a 0xd46704 0xd47bfe 0xd44f4d 0x8f055c 0xc1c4ac 0xc1a483 0x98e9fd 0x98f11a 0x9908b4 0x990fb5 0x99306d 0x9b31c4 0x9b4103 0x9b4bbb 0xb80ea5 0xa941fa 0x7e5320 0xee37d1 0x7fd727974e9a 0x7fd726a842ed ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B25749"},{"b":"400000","o":"ACF571"},{"b":"400000","o":"AB629A"},{"b":"400000","o":"946704"},{"b":"400000","o":"947BFE"},{"b":"400000","o":"944F4D"},{"b":"400000","o":"4F055C"},{"b":"400000","o":"81C4AC"},{"b":"400000","o":"81A483"},{"b":"400000","o":"58E9FD"},{"b":"400000","o":"58F11A"},{"b":"400000","o":"5908B4"},{"b":"400000","o":"590FB5"},{"b":"400000","o":"59306D"},{"b":"400000","o":"5B31C4"},{"b":"400000","o":"5B4103"},{"b":"400000","o":"5B4BBB"},{"b":"400000","o":"780EA5"},{"b":"400000","o":"6941FA"},{"b":"400000","o":"3E5320"},{"b":"400000","o":"AE37D1"},{"b":"7FD72796D000","o":"7E9A"},{"b":"7FD726990000","o":"F42ED"}],"processInfo":{ "mongodbVersion" : "2.8.0-rc5", "gitVersion" : "74b351de21c84438b12a83b28e155f5e69e3c1eb", "uname" : { "sysname" : "Linux", "release" : "3.2.0-74-generic", "version" : "#109-Ubuntu SMP Tue Dec 9 16:45:49 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFF120FF000", "elfType" : 3 }, { "b" : "7FD72796D000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }, { "b" : "7FD727765000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7FD727561000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }, { "b" : "7FD727261000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7FD726F65000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }, { "b" : "7FD726D4F000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7FD726990000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }, { "b" : "7FD727B8A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf25749] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xecf571] mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0xDA) [0xeb629a] mongod(_ZN5mongo21WiredTigerRecordStore20cappedDeleteAsNeededEPNS_16OperationContextERKNS_8RecordIdE+0x5F4) [0xd46704] mongod(_ZN5mongo21WiredTigerRecordStore12insertRecordEPNS_16OperationContextEPKcib+0x1DE) [0xd47bfe] mongod(_ZN5mongo21WiredTigerRecordStore12insertRecordEPNS_16OperationContextEPKNS_9DocWriterEb+0x8D) [0xd44f4d] mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextEPKNS_9DocWriterEb+0x5C) [0x8f055c] mongod(+0x81C4AC) [0xc1c4ac] mongod(_ZN5mongo4repl5logOpEPNS_16OperationContextEPKcS4_RKNS_7BSONObjEPS5_Pbb+0xA3) [0xc1a483] mongod(_ZN5mongo18WriteBatchExecutor13execOneInsertEPNS0_16ExecInsertsStateEPPNS_16WriteErrorDetailE+0xEED) [0x98e9fd] mongod(_ZN5mongo18WriteBatchExecutor11execInsertsERKNS_21BatchedCommandRequestEPSt6vectorIPNS_16WriteErrorDetailESaIS6_EE+0x25A) [0x98f11a] mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestEPSt6vectorIPNS_19BatchedUpsertDetailESaIS6_EEPS4_IPNS_16WriteErrorDetailESaISB_EE+0x34) [0x9908b4] mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x395) [0x990fb5] mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x15D) [0x99306d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9b31c4] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC13) [0x9b4103] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9b4bbb] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_b+0x755) [0xb80ea5] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortEb+0xB0A) [0xa941fa] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xE0) [0x7e5320] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x321) [0xee37d1] libpthread.so.0(+0x7E9A) [0x7fd727974e9a] libc.so.6(clone+0x6D) [0x7fd726a842ed] ----- END BACKTRACE ----- 2015-01-28T08:40:01.457+0100 I - [conn76]
- related to
-
SERVER-17141 session->truncate failed with File Not Found in WiredTigerRecordStoreThread for local.oplog.rs
- Closed