2015-10-28T10:05:26.877-0500 I NETWORK [initandlisten] connection accepted from application_ip:43070 #36 (36 connections now open) 2015-10-28T10:05:27.177-0500 I REPL [ReplicationExecutor] syncing from: secondary.mongo:27017 2015-10-28T10:05:27.204-0500 I REPL [SyncSourceFeedback] replset setting syncSourceFeedback to secondary.mongo:27017 2015-10-28T10:05:27.219-0500 E STORAGE [repl writer worker 15] WiredTiger (0) [1446044727:219672][13821:0x7f4985017700], file:index-1105--2021113240517181031.wt, session.open_cursor: encountered an illegal file format or internal value 2015-10-28T10:05:27.219-0500 E STORAGE [repl writer worker 15] WiredTiger (-31804) [1446044727:219746][13821:0x7f4985017700], file:index-1105--2021113240517181031.wt, session.open_cursor: the process must exit and restart: WT_PANIC: WiredTiger library panic 2015-10-28T10:05:27.219-0500 I - [repl writer worker 15] Fatal Assertion 28558 2015-10-28T10:05:27.245-0500 I CONTROL [repl writer worker 15] 0xf5ba59 0xefa911 0xedf501 0xd8577a 0x1390219 0x13903d5 0x1390874 0x12fffd4 0x12fae7c 0x12fbca2 0x132582b 0x138f690 0x138f84a 0x1332c51 0x138d7ba 0x1344393 0x138d601 0x138da15 0xd7eae2 0xd7ac8b 0xd69075 0xa82695 0xa0b740 0xa377fc 0xbce204 0xbce5d4 0xbcec3d 0xb1f248 0xc38bb8 0xc9e2ef 0xca1195 0xef172b 0xfa9724 0x7f499a0d0182 0x7f4998b8247d ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B5BA59"},{"b":"400000","o":"AFA911"},{"b":"400000","o":"ADF501"},{"b":"400000","o":"98577A"},{"b":"400000","o":"F90219"},{"b":"400000","o":"F903D5"},{"b":"400000","o":"F90874"},{"b":"400000","o":"EFFFD4"},{"b":"400000","o":"EFAE7C"},{"b":"400000","o":"EFBCA2"},{"b":"400000","o":"F2582B"},{"b":"400000","o":"F8F690"},{"b":"400000","o":"F8F84A"},{"b":"400000","o":"F32C51"},{"b":"400000","o":"F8D7BA"},{"b":"400000","o":"F44393"},{"b":"400000","o":"F8D601"},{"b":"400000","o":"F8DA15"},{"b":"400000","o":"97EAE2"},{"b":"400000","o":"97AC8B"},{"b":"400000","o":"969075"},{"b":"400000","o":"682695"},{"b":"400000","o":"60B740"},{"b":"400000","o":"6377FC"},{"b":"400000","o":"7CE204"},{"b":"400000","o":"7CE5D4"},{"b":"400000","o":"7CEC3D"},{"b":"400000","o":"71F248"},{"b":"400000","o":"838BB8"},{"b":"400000","o":"89E2EF"},{"b":"400000","o":"8A1195"},{"b":"400000","o":"AF172B"},{"b":"400000","o":"BA9724"},{"b":"7F499A0C8000","o":"8182"},{"b":"7F4998A88000","o":"FA47D"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "3.13.0-61-generic", "version" : "#100-Ubuntu SMP Wed Jul 29 11:21:34 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "695FC6828398A9DB1F99718671147885B5ED116D" }, { "b" : "7FFDA40A8000", "elfType" : 3, "buildId" : "A3F2043AB5D2D7974286088B9241BE9E21C44687" }, { "b" : "7F499A0C8000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "9318E8AF0BFBE444731BB0461202EF57F7C39542" }, { "b" : "7F4999E68000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "A20EFFEC993A8441FA17F2079F923CBD04079E19" }, { "b" : "7F4999A88000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "F000D29917E9B6E94A35A8F02E5C62846E5916BC" }, { "b" : "7F4999880000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "92FCF41EFE012D6186E31A59AD05BDBB487769AB" }, { "b" : "7F4999678000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "C1AE4CB7195D337A77A3C689051DABAA3980CA0C" }, { "b" : "7F4999370000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3, "buildId" : "4BF6F7ADD8244AD86008E6BF40D90F8873892197" }, { "b" : "7F4999068000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "1D76B71E905CB867B27CEF230FCB20F01A3178F5" }, { "b" : "7F4998E50000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "8D0AA71411580EE6C08809695C3984769F25725B" }, { "b" : "7F4998A88000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "30C94DC66A1FE95180C3D68D2B89E576D5AE213C" }, { "b" : "7F499A2E8000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9F00581AB3C73E3AEA35995A0C50D24D59A01D47" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf5ba59] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xefa911] mongod(_ZN5mongo13fassertFailedEi+0x61) [0xedf501] mongod(+0x98577A) [0xd8577a] mongod(__wt_eventv+0x489) [0x1390219] mongod(__wt_err+0x95) [0x13903d5] mongod(__wt_panic+0x24) [0x1390874] mongod(__wt_page_inmem+0xB04) [0x12fffd4] mongod(__wt_btree_tree_open+0x8C) [0x12fae7c] mongod(__wt_btree_open+0xD02) [0x12fbca2] mongod(__wt_conn_btree_get+0x19B) [0x132582b] mongod(__wt_session_get_btree+0x500) [0x138f690] mongod(__wt_session_get_btree_ckpt+0xBA) [0x138f84a] mongod(__wt_curfile_open+0xE1) [0x1332c51] mongod(__wt_open_cursor+0x26A) [0x138d7ba] mongod(__wt_curtable_open+0x2E3) [0x1344393] mongod(__wt_open_cursor+0xB1) [0x138d601] mongod(+0xF8DA15) [0x138da15] mongod(_ZN5mongo17WiredTigerSession9getCursorERKSsmb+0x92) [0xd7eae2] mongod(_ZN5mongo16WiredTigerCursorC1ERKSsmbPNS_16OperationContextE+0x4B) [0xd7ac8b] mongod(_ZNK5mongo21WiredTigerIndexUnique9newCursorEPNS_16OperationContextEi+0x55) [0xd69075] mongod(_ZNK5mongo22BtreeBasedAccessMethod10findSingleEPNS_16OperationContextERKNS_7BSONObjE+0x25) [0xa82695] mongod(_ZN5mongo11IDHackStage4workEPm+0xD0) [0xa0b740] mongod(_ZN5mongo11UpdateStage4workEPm+0x7C) [0xa377fc] mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0xA4) [0xbce204] mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x34) [0xbce5d4] mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbcec3d] mongod(_ZN5mongo6updateEPNS_16OperationContextEPNS_8DatabaseERKNS_13UpdateRequestEPNS_7OpDebugE+0x118) [0xb1f248] mongod(_ZN5mongo4repl21applyOperation_inlockEPNS_16OperationContextEPNS_8DatabaseERKNS_7BSONObjEbb+0x1308) [0xc38bb8] mongod(_ZN5mongo4repl8SyncTail9syncApplyEPNS_16OperationContextERKNS_7BSONObjEb+0x2EF) [0xc9e2ef] mongod(_ZN5mongo4repl14multiSyncApplyERKSt6vectorINS_7BSONObjESaIS2_EEPNS0_8SyncTailE+0x65) [0xca1195] mongod(_ZN5mongo10threadpool6Worker4loopERKSs+0x2FB) [0xef172b] mongod(+0xBA9724) [0xfa9724] libpthread.so.0(+0x8182) [0x7f499a0d0182] libc.so.6(clone+0x6D) [0x7f4998b8247d] ----- END BACKTRACE ----- 2015-10-28T10:05:27.245-0500 I - [repl writer worker 15] ***aborting after fassert() failure