-
Type:
Bug
-
Resolution: Duplicate
-
Priority:
Major - P3
-
None
-
Affects Version/s: 3.0.3
-
Component/s: WiredTiger
-
None
-
Environment:Ubuntu xnu-2782.20.48~5/RELEASE_X86_64 x86_64
-
Linux
-
Our Deployment Environment:
=======================
Shared Replica Set.
Ubuntu Machine.
Everything was running fine until WiredTiger Upgrade. We have 4 shards per Linux Instance. Upgraded from MMAP to WiredTiger i.e. Deleted all the data and let it sync with Primary and during sync we are getting Out Of Memory Error and the mongod process running for that shard exits.
We have a 15GB Memory Instance so we assumed 3 GB reserved memory for other system processes and for each mongod instance running per shard we set cacheSizeGB to 3 GB.
Error Stack Trace:
================
2015-06-22T08:42:22.007+0000 I - [rsSync] Index Build: 1439100/92773983 1%
2015-06-22T08:42:24.176+0000 F - [rsSync] out of memory.
0xf51949 0xf51479 0x13f5a0f 0x7f7e3fa373b9 0xd3d251 0x7f7e3fa389ec 0xa95ea6 0xa9c478 0xa9d791 0xa98914 0xa8809a 0x92d0a5 0x92e21a 0x9380d0 0x93a326 0xc84036 0xc854ca 0xc85f82 0xc8ea81 0xf9f9b4 0x7f7e406cd182 0x7f7e3f19630d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B51949"},{"b":"400000","o":"B51479"},{"b":"400000","o":"FF5A0F"},{"b":"7F7E3F97D000","o":"BA3B9"},{"b":"400000","o":"93D251"},{"b":"7F7E3F97D000","o":"BB9EC"},{"b":"400000","o":"695EA6"},{"b":"400000","o":"69C478"},{"b":"400000","o":"69D791"},{"b":"400000","o":"698914"},{"b":"400000","o":"68809A"},{"b":"400000","o":"52D0A5"},{"b":"400000","o":"52E21A"},{"b":"400000","o":"5380D0"},{"b":"400000","o":"53A326"},{"b":"400000","o":"884036"},{"b":"400000","o":"8854CA"},{"b":"400000","o":"885F82"},{"b":"400000","o":"88EA81"},{"b":"400000","o":"B9F9B4"},{"b":"7F7E406C5000","o":"8182"},{"b":"7F7E3F09B000","o":"FB30D"}],"processInfo":{ "mongodbVersion" : "3.0.3", "gitVersion" : "b40106b36eecd1b4407eb1ad1af6bc60593c6105", "uname" : { "sysname" : "Linux", "release" : "3.13.0-29-generic", "version" : "#53-Ubuntu SMP Wed Jun 4 21:00:20 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "F56F80CB96B4DBFC070BEB0ADAC7D6B274BFC6B1" }, { "b" : "7FFF436FE000", "elfType" : 3, "buildId" : "3D068D088E7EAC15D9DA7C3AC912E783C0897EE7" }, { "b" : "7F7E406C5000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "FE662C4D7B14EE804E0C1902FB55218A106BC5CB" }, { "b" : "7F7E40467000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "6C7AE380840DB9034D7763771B55E51B31BCAF14" }, { "b" : "7F7E4008D000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "3D522D8E04F5FD7904AE69B50CA8835A71024490" }, { "b" : "7F7E3FE85000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "92FCF41EFE012D6186E31A59AD05BDBB487769AB" }, { "b" : "7F7E3FC81000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "C1AE4CB7195D337A77A3C689051DABAA3980CA0C" }, { "b" : "7F7E3F97D000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3, "buildId" : "19EFDDAB11B3BF5C71570078C59F91CF6592CE9E" }, { "b" : "7F7E3F677000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "574C6350381DA194C00FF555E0C1784618C05569" }, { "b" : "7F7E3F461000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "CC0D578C2E0D86237CA7B0CE8913261C506A629A" }, { "b" : "7F7E3F09B000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B571F83A8A6F5BB22D3558CDDDA9F943A2A67FD1" }, { "b" : "7F7E408E3000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9F00581AB3C73E3AEA35995A0C50D24D59A01D47" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf51949]
mongod(_ZN5mongo29reportOutOfMemoryErrorAndExitEv+0x49) [0xf51479]
mongod(tc_new+0x1AF) [0x13f5a0f]
libstdc++.so.6(_ZNSs4_Rep9_S_createEmmRKSaIcE+0x59) [0x7f7e3fa373b9]
mongod(_ZNSs12_S_constructIPcEES0_T_S1_RKSaIcESt20forward_iterator_tag+0x21) [0xd3d251]
libstdc++.so.6(_ZNSsC1ERKSsmm+0x3C) [0x7f7e3fa389ec]
mongod(_ZNK5mongo19BtreeKeyGeneratorV118extractNextElementERKNS_7BSONObjES3_RPKcRb+0x66) [0xa95ea6]
mongod(_ZNK5mongo19BtreeKeyGeneratorV120getKeysImplWithArrayESt6vectorIPKcSaIS3_EES1_INS_11BSONElementESaIS6_EERKNS_7BSONObjEPSt3setIS9_NS_10BSONObjCmpESaIS9_EEjSB_+0x118) [0xa9c478]
mongod(_ZNK5mongo19BtreeKeyGeneratorV111getKeysImplESt6vectorIPKcSaIS3_EES1_INS_11BSONElementESaIS6_EERKNS_7BSONObjEPSt3setIS9_NS_10BSONObjCmpESaIS9_EE+0x151) [0xa9d791]
mongod(_ZNK5mongo17BtreeKeyGenerator7getKeysERKNS_7BSONObjEPSt3setIS1_NS_10BSONObjCmpESaIS1_EE+0x244) [0xa98914]
mongod(_ZN5mongo26BtreeBasedBulkAccessMethod6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdERKNS_19InsertDeleteOptionsEPl+0x6A) [0xa8809a]
mongod(_ZN5mongo15MultiIndexBlock6insertERKNS_7BSONObjERKNS_8RecordIdE+0x75) [0x92d0a5]
mongod(_ZN5mongo15MultiIndexBlock30insertAllDocumentsInCollectionEPSt3setINS_8RecordIdESt4lessIS2_ESaIS2_EE+0x20A) [0x92e21a]
mongod(_ZN5mongo6Cloner11copyIndexesEPNS_16OperationContextERKSsRKNS_15NamespaceStringES7_bbbbb+0x470) [0x9380d0]
mongod(_ZN5mongo6Cloner2goEPNS_16OperationContextERKSsS4_RKNS_12CloneOptionsEPSt3setISsSt4lessISsESaISsEERSsPi+0x1106) [0x93a326]
mongod(+0x884036) [0xc84036]
mongod(+0x8854CA) [0xc854ca]
mongod(_ZN5mongo4repl17syncDoInitialSyncEv+0x42) [0xc85f82]
mongod(_ZN5mongo4repl13runSyncThreadEv+0x181) [0xc8ea81]
mongod(+0xB9F9B4) [0xf9f9b4]
libpthread.so.0(+0x8182) [0x7f7e406cd182]
libc.so.6(clone+0x6D) [0x7f7e3f19630d]
----- END BACKTRACE -----
- duplicates
-
-
- Closed
-