Hi Mongo Team, 

Prod mongo instance secondary node crashed and got below errors:

 Mongo Verson ： 4.0.13

 OS: Red Hat Enterprise Linux Server release 7.7 (Maipo)

--------------------------------------------------------------------------- 

2021-05-26T00:19:18.520+0800 I STORAGE [WT OplogTruncaterThread: local.oplog.rs] WiredTiger record store oplog truncation finished in: 16019ms
2021-05-26T00:21:58.751+0800 F - [thread1990485] Invalid access at address: 0
2021-05-26T00:21:58.862+0800 F - [thread1990485] Got signal: 11 (Segmentation fault).
0x561548c2aa31 0x561548c29c49 0x561548c2a2b6 0x7f238cae1630 0x561548d17b53 0x561548d17c2c 0x561548dba42a 0x561547397916 0x561547397996 0x561547398460 0x5615473b1b7b 0x561547329339 0x561547321e64 0x5615473223d3 0x561547324727 0x561547373079 0x7f238cad9ea5 0x7f238c8028cd
----- BEGIN BACKTRACE -----

{"backtrace":[\{"b":"5615467D7000","o":"2453A31","s":"_ZN5mongo15printStackTraceERSo"}

,{"b":"5615467D7000","o":"2452C49"},{"b":"5615467D7000","o":"24532B6"},{"b":"7F238CAD2000","o":"F630"},{"b":"5615467D7000","o":"2540B53","s":"ZN8tcmalloc11ThreadCache21ReleaseToCentralCacheEPNS0_8FreeListEmi"},{"b":"5615467D7000","o":"2540C2C","s":"_ZN8tcmalloc11ThreadCache11ListTooLongEPNS0_8FreeListEm"},{"b":"5615467D7000","o":"25E342A","s":"_ZdlPvRKSt9nothrow_t"},{"b":"5615467D7000","o":"BC0916"},{"b":"5615467D7000","o":"BC0996"},{"b":"5615467D7000","o":"BC1460","s":"wt_page_out"},{"b":"5615467D7000","o":"BDAB7B","s":"wt_split_rewrite"},{"b":"5615467D7000","o":"B52339","s":"wt_evict"},{"b":"5615467D7000","o":"B4AE64"},{"b":"5615467D7000","o":"B4B3D3"},{"b":"5615467D7000","o":"B4D727","s":"_wt_evict_thread_run"},{"b":"5615467D7000","o":"B9C079"},{"b":"7F238CAD2000","o":"7EA5"},{"b":"7F238C704000","o":"FE8CD","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.13", "gitVersion" : "bda366f0b0e432ca143bc41da54d8732bd8d03c0", "compiledModules" : [], "uname" :

{ "sysname" : "Linux", "release" : "4.14.35-1902.300.11.el7uek.x86_64", "version" : "#2 SMP Tue Mar 17 17:11:47 PDT 2020", "machine" : "x86_64" }

, "somap" : [ { "b" : "5615467D7000", "elfType" : 3, "buildId" : "41D0BDEE62883413FD0B29F34271F4BAF38ED244" }, { "b" : "7FFE0C0FC000", "elfType" : 3, "buildId" : "51107E6B28984C2BDA66262EB5FA1FD7C263C43C" }, { "b" : "7F238DF00000", "path" : "/lib64/libcurl.so.4", "elfType" : 3, "buildId" : "328BC5C8C1C0E3CA337CF1A5143FB9EFF6F69920" }, { "b" : "7F238DCE7000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "704764A32A008A3893169AFC3A10273C680ACA79" }, { "b" : "7F238D884000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "4CF1939F660008CFA869D8364651F31AACD2C1C4" }, { "b" : "7F238D612000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "3B305C3BA17FE394862E749763F2956C9C890C2E" }, { "b" : "7F238D40E000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "A8E5F1C9B2954D62BE6DC99A4AA353F5B0FA502E" }, { "b" : "7F238D206000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "2ECEA19D68022C5DD8C19C2F423C16A2281564C4" }, { "b" : "7F238CF04000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "0FE1F06D15F044AD4A71A6D41827C06606A24342" }, { "b" : "7F238CCEE000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "381ABC3B2D1309B4E9CA1846E6D6FEBB45BCD82A" }, { "b" : "7F238CAD2000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "7C7FFC7DA2A88110FCE29E5DBC2DA3084E05EA08" }, { "b" : "7F238C704000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "2935718B7F1F0B2CB52A9C53D1B5DF375FB848D7" }, { "b" : "7F238E16A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "1455E4B275FFC75EB36876743DC9D1AFAD89A652" }, { "b" : "7F238C4D1000", "path" : "/lib64/libidn.so.11", "elfType" : 3, "buildId" : "DC7CCADE5A92D80542C6548F2D5ECCCC0E8D9CE0" }, { "b" : "7F238C2A4000", "path" : "/lib64/libssh2.so.1", "elfType" : 3, "buildId" : "C4252D7B84334B43BA9C66059DF0FBD1525EE0E4" }, { "b" : "7F238C04B000", "path" : "/lib64/libssl3.so", "elfType" : 3, "buildId" : "DA5AD82CCD1571AE123C4CE7BBA98DF3C8268B3B" }, { "b" : "7F238BE23000", "path" : "/lib64/libsmime3.so", "elfType" : 3, "buildId" : "96B8CB67C43544981373F6C020FBFDDB26CC86C9" }, { "b" : "7F238BAF4000", "path" : "/lib64/libnss3.so", "elfType" : 3, "buildId" : "8E4067E346DDD3BDB410A5B113AFF5CF0490C36A" }, { "b" : "7F238B8C4000", "path" : "/lib64/libnssutil3.so", "elfType" : 3, "buildId" : "8110BD747F9BD1E0B85A409A8EFF63CAB3205CC6" }, { "b" : "7F238B6C0000", "path" : "/lib64/libplds4.so", "elfType" : 3, "buildId" : "8003A511613AB114927A2C349D62BC89B9372F2D" }, { "b" : "7F238B4BB000", "path" : "/lib64/libplc4.so", "elfType" : 3, "buildId" : "62CCA75DB229A4D9C5B92F7CC36CE91DDDF37CF8" }, { "b" : "7F238B27D000", "path" : "/lib64/libnspr4.so", "elfType" : 3, "buildId" : "C156A0EB8916E598D9A73E5FDBECE866E69EF28C" }, { "b" : "7F238B030000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "374C0C6A0AB18FD797D4463120984A9808E210F4" }, { "b" : "7F238AD47000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "2CBF129F383744C38FB58F168382903BD5FE0830" }, { "b" : "7F238AB14000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "2B5920105B622A53B643DD695AF6ED3890711205" }, { "b" : "7F238A910000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "51300757BCF41349AB8163A79460062664AF2FE8" }, { "b" : "7F238A701000", "path" : "/lib64/liblber-2.4.so.2", "elfType" : 3, "buildId" : "129E26CE082C75D77BB516FA83D301BEBD7134C2" }, { "b" : "7F238A4AC000", "path" : "/lib64/libldap-2.4.so.2", "elfType" : 3, "buildId" : "5F428CF33D73BE851173294EE9B55BD8130D0359" }, { "b" : "7F238A296000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "B9D5F73428BD6AD68C96986B57BEA3B7CEDB9745" }, { "b" : "7F238A086000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "AB414564AF9F922AB798C44A8F82276F113E7BA2" }, { "b" : "7F2389E82000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "3B0403FDFDE98D24FA2B5EA33202259FADF9E9A1" }, { "b" : "7F2389C65000", "path" : "/lib64/libsasl2.so.3", "elfType" : 3, "buildId" : "9AF2AD92DADE046C6260DCCF02846BF78ABC658C" }, { "b" : "7F2389A3E000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "D2DD4DA3FDE1477D25BFFF80F3A25FDB541A8179" }, { "b" : "7F2389807000", "path" : "/lib64/libcrypt.so.1", "elfType" : 3, "buildId" : "8B08A419D1DF08C18154B4229E803183737F1AA0" }, { "b" : "7F23895A5000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "F5B144F9F5D9BE451C80211B34DB2CE348E039B6" }, { "b" : "7F23893A2000", "path" : "/lib64/libfreebl3.so", "elfType" : 3, "buildId" : "FAB7DDA6F331895E419D648989BF9BE6B4C3B9B3" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x561548c2aa31]
mongod(+0x2452C49) [0x561548c29c49]
mongod(+0x24532B6) [0x561548c2a2b6]
libpthread.so.0(+0xF630) [0x7f238cae1630]
mongod(_ZN8tcmalloc11ThreadCache21ReleaseToCentralCacheEPNS0_8FreeListEmi+0xE3) [0x561548d17b53]
mongod(_ZN8tcmalloc11ThreadCache11ListTooLongEPNS0_8FreeListEm+0x1C) [0x561548d17c2c]
mongod(_ZdlPvRKSt9nothrow_t+0x26A) [0x561548dba42a]
mongod(+0xBC0916) [0x561547397916]
mongod(+0xBC0996) [0x561547397996]
mongod(__wt_page_out+0x5E0) [0x561547398460]
mongod(__wt_split_rewrite+0x2FB) [0x5615473b1b7b]
mongod(__wt_evict+0x1539) [0x561547329339]
mongod(+0xB4AE64) [0x561547321e64]
mongod(+0xB4B3D3) [0x5615473223d3]
mongod(__wt_evict_thread_run+0x77) [0x561547324727]
mongod(+0xB9C079) [0x561547373079]
libpthread.so.0(+0x7EA5) [0x7f238cad9ea5]
libc.so.6(clone+0x6D) [0x7f238c8028cd]
----- END BACKTRACE -----

Could you please help investgation? 

 

 

Thanks

Jason