Hi Team,

Please find below MongoDB incident that occurred on 10:40 PM, 21 Sept 2021

Event Description:
We received a mongo process down alert on primary DB at 10:40 PM, 21 Sept 2021

Chronology of Events/Timeline:

10:40 PM: Received alerts regarding Mongo process down on master server(172.31.5.61), OpsGenie Alert

10:51 PM: Started Mongo process  manually on the master server (172.31.5.61).

10:55 PM: We verified all mongo processes are working fine and identified that master server failover to slave server (172.31.5.52)

 

Error Found on mongo log:

2021-09-21T22:37:13.898+0530 E STORAGE [conn56586] WiredTiger error (22) [1632244033:898964][1411:0x7f18a5c77700], WT_SESSION.timestamp_transaction: __wt_txn_set_commit_timestamp, 683: commit timestamp (1632244033, 1048) is less than the oldest timestamp (1632244033, 1051): Invalid argument Raw: [1632244033:898964][1411:0x7f18a5c77700], WT_SESSION.timestamp_transaction: __wt_txn_set_commit_timestamp, 683: commit timestamp (1632244033, 1048) is less than the oldest timestamp (1632244033, 1051): Invalid argument
2021-09-21T22:37:13.899+0530 F - [conn56586] Fatal assertion 39001 BadValue: timestamp_transaction 22: Invalid argument at src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp 1323
2021-09-21T22:37:13.899+0530 F - [conn56586]

***aborting after fassert() failure

2021-09-21T22:37:13.966+0530 F - [conn56586] Got signal: 6 (Aborted).
0x556ee1ec4bf1 0x556ee1ec420c 0x556ee1ec4296 0x7f18cc0477e0 0x7f18cbcbec20 0x7f18cbcc00c8 0x556ee031aba1 0x556ee0053d4d 0x556ee0418f2c 0x556ee0be67e1 0x556ee0c7b613 0x556ee0c7fdac 0x556ee0a67dbb 0x556ee0a6ddbc 0x556ee03a6107 0x556ee0becc8c 0x556ee0dabb2d 0x556ee0dad212 0x556ee0d8b908 0x556ee0dd2e30 0x556ee0dd358d 0x556ee0a988e7 0x556ee0a9a6fb 0x556ee0a9b96a 0x556ee18b3cef 0x556ee079158a 0x556ee0792875 0x556ee07948f4 0x556ee079568a 0x556ee078320c 0x556ee078ef9c 0x556ee078c27f 0x556ee078df0c 0x556ee15e9f82 0x556ee078914d 0x556ee078a9c3 0x556ee078b576 0x556ee078c1db 0x556ee078df0c 0x556ee15ea3eb 0x556ee1c3ff35 0x556ee1c3ff94 0x7f18cc03d40b 0x7f18cbd7809f
----- BEGIN BACKTRACE -----

{"backtrace":[

{"b":"556EDF606000","o":"28BEBF1","s":"_ZN5mongo15printStackTraceERSo"}

,{"b":"556EDF606000","o":"28BE20C"},{"b":"556EDF606000","o":"28BE296"},{"b":"7F18CC036000","o":"117E0"},{"b":"7F18CBC8B000","o":"33C20","s":"gsignal"},{"b":"7F18CBC8B000","o":"350C8","s":"abort"},{"b":"556EDF606000","o":"D14BA1","s":"_ZN5mongo42fassertFailedWithStatusNoTraceWithLocationEiRKNS_6StatusEPKcj"},{"b":"556EDF606000","o":"A4DD4D"},{"b":"556EDF606000","o":"E12F2C","s":"_ZN5mongo21WiredTigerRecordStore26insertRecordsWithDocWriterEPNS_16OperationContextEPKPKNS_9DocWriterEPKNS_9TimestampEmPNS_8RecordIdE"},{"b":"556EDF606000","o":"15E07E1","s":"_ZN5mongo14CollectionImpl23insertDocumentsForOplogEPNS_16OperationContextEPKPKNS_9DocWriterEPNS_9TimestampEm"},{"b":"556EDF606000","o":"1675613","s":"_ZN5mongo4repl12_logOpsInnerEPNS_16OperationContextERKNS_15NamespaceStringEPKPKNS_9DocWriterEPNS_9TimestampEmPNS_10CollectionENS0_6OpTimeENS_6Date_tE"},{"b":"556EDF606000","o":"1679DAC","s":"_ZN5mongo4repl5logOpEPNS_16OperationContextEPKcRKNS_15NamespaceStringEN5boost8optionalINS_4UUIDEEERKNS_7BSONObjEPSD_bNS_6Date_tERKNS_20OperationSessionInfoENS9_IiEERKNS0_9OplogLinkERKNS0_6OpTimeE"},{"b":"556EDF606000","o":"1461DBB"},{"b":"556EDF606000","o":"1467DBC","s":"_ZN5mongo14OpObserverImpl8onUpdateEPNS_16OperationContextERKNS_20OplogUpdateEntryArgsE"},{"b":"556EDF606000","o":"DA0107","s":"_ZN5mongo18OpObserverRegistry8onUpdateEPNS_16OperationContextERKNS_20OplogUpdateEntryArgsE"},{"b":"556EDF606000","o":"15E6C8C","s":"_ZN5mongo14CollectionImpl25updateDocumentWithDamagesEPNS_16OperationContextENS_8RecordIdERKNS_11SnapshottedINS_10RecordDataEEEPKcRKSt6vectorINS_11mutablebson11DamageEventESaISD_EEPNS_20CollectionUpdateArgsE"},{"b":"556EDF606000","o":"17A5B2D","s":"_ZN5mongo11UpdateStage18transformAndUpdateERKNS_11SnapshottedINS_7BSONObjEEERNS_8RecordIdE"},{"b":"556EDF606000","o":"17A7212","s":"_ZN5mongo11UpdateStage6doWorkEPm"},{"b":"556EDF606000","o":"1785908","s":"_ZN5mongo9PlanStage4workEPm"},{"b":"556EDF606000","o":"17CCE30","s":"_ZN5mongo16PlanExecutorImpl12_getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"556EDF606000","o":"17CD58D","s":"_ZN5mongo16PlanExecutorImpl7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"556EDF606000","o":"14928E7"},{"b":"556EDF606000","o":"14946FB"},{"b":"556EDF606000","o":"149596A"},{"b":"556EDF606000","o":"22ADCEF","s":"_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_3rpc21ReplyBuilderInterfaceE"},{"b":"556EDF606000","o":"118B58A"},{"b":"556EDF606000","o":"118C875"},{"b":"556EDF606000","o":"118E8F4"},{"b":"556EDF606000","o":"118F68A","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE"},{"b":"556EDF606000","o":"117D20C","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"556EDF606000","o":"1188F9C","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"556EDF606000","o":"118627F","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"556EDF606000","o":"1187F0C"},{"b":"556EDF606000","o":"1FE3F82","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE"},{"b":"556EDF606000","o":"118314D","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE"},{"b":"556EDF606000","o":"11849C3","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"556EDF606000","o":"1185576","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"556EDF606000","o":"11861DB","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"556EDF606000","o":"1187F0C"},{"b":"556EDF606000","o":"1FE43EB"},{"b":"556EDF606000","o":"2639F35"},{"b":"556EDF606000","o":"2639F94"},{"b":"7F18CC036000","o":"740B"},{"b":"7F18CBC8B000","o":"ED09F","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.2.8", "gitVersion" : "43d25964249164d76d5e04dd6cf38f6111e21f5f", "compiledModules" : [], "uname" :

{ "sysname" : "Linux", "release" : "4.14.231-173.361.amzn2.x86_64", "version" : "#1 SMP Mon Apr 26 20:57:08 UTC 2021", "machine" : "x86_64" }

, "somap" : [ { "b" : "556EDF606000", "elfType" : 3, "buildId" : "1118CB24C802841614B9E8C04394950FD56BD9C3" }, { "b" : "7FFD59FA8000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "FD7ED28E435B892351687132649AD89F7DDB810E" }, { "b" : "7F18CD491000", "path" : "/lib64/libcurl.so.4", "elfType" : 3, "buildId" : "B737D249E957BD15A72E1E0819C56A24EBF6E2F3" }, { "b" : "7F18CD27B000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "D54CF97BF07398E614DFCC7C91BCFC5E9071A947" }, { "b" : "7F18CCE26000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "6A0BDB33426B1355207BF1450AAE789918AE14CC" }, { "b" : "7F18CCBB7000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "C589FBDB14C2DB872D9A9BE1B7692A40E25B35DE" }, { "b" : "7F18CC9B3000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "765CB7C0C3F23012206E28836DA842B4E26CEA32" }, { "b" : "7F18CC7AB000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "73AAF1068EFB9CDB2C5D7B30A3737A3632BD8BD6" }, { "b" : "7F18CC46B000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "2EDF81157EE48F2E86E342A8129249A69FFB6358" }, { "b" : "7F18CC254000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "025CC3324B759F1D51215F04B7976A87BA655534" }, { "b" : "7F18CC036000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "FBB1EBCD3AA68F9FF71EAFF037EC3F0EB1FE45B9" }, { "b" : "7F18CBC8B000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "8779C8D644555E9918F28FB0B801AEA29A6692B8" }, { "b" : "7F18CD716000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "BB7472F0482DB35A29453A96BEE279DAF83C0ADB" }, { "b" : "7F18CBA64000", "path" : "/lib64/libnghttp2.so.14", "elfType" : 3, "buildId" : "ABE2627C22FDC7706352089D224CA49759CB74F9" }, { "b" : "7F18CB817000", "path" : "/lib64/libidn2.so.0", "elfType" : 3, "buildId" : "F570AD4111FD2ADDA7A7052E4A8ADB2427D62703" }, { "b" : "7F18CB5EF000", "path" : "/lib64/libssh2.so.1", "elfType" : 3, "buildId" : "DB7387A967A1DB497B643672C4E4DA5A9931D347" }, { "b" : "7F18CB3A2000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "B66D7C7DE3D59F794AD60D2B71482E17484CF860" }, { "b" : "7F18CB0BA000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "9677E50188E2ADDBED7302D36BECA8E1F616E77E" }, { "b" : "7F18CAE88000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "4D65FAB419F439A47361C55866A78A9768774517" }, { "b" : "7F18CAC84000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "617A0D6614294A0460D068C62AC3EEB5D6E93A25" }, { "b" : "7F18CAA32000", "path" : "/lib64/libldap-2.4.so.2", "elfType" : 3, "buildId" : "203805A61C19203C8042A8360F3908C34DA43E4B" }, { "b" : "7F18CA823000", "path" : "/lib64/liblber-2.4.so.2", "elfType" : 3, "buildId" : "2FFCC7CCB69677D18DD4EEAEFBACAB966303539D" }, { "b" : "7F18CA60D000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "B3D7471B1072E67716DF857D800318A7F47A09FB" }, { "b" : "7F18CA2F5000", "path" : "/lib64/libunistring.so.0", "elfType" : 3, "buildId" : "058ACB5009C38557BD075015D6D55DFCB0118EB4" }, { "b" : "7F18CA0E6000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E35E6A8E73183AD3B40E8A21574E1D2780ADB599" }, { "b" : "7F18C9EE2000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "895D9F7FAA8E8ADCF110AA559D251258EFD4C924" }, { "b" : "7F18C9CC5000", "path" : "/lib64/libsasl2.so.3", "elfType" : 3, "buildId" : "5E6E0F28762D9135F24BAD9D1B2723833DFE4C29" }, { "b" : "7F18C9A6B000", "path" : "/lib64/libssl3.so", "elfType" : 3, "buildId" : "366998CC2B48C772F744CA5CA8E5253ED665EE3F" }, { "b" : "7F18C9845000", "path" : "/lib64/libsmime3.so", "elfType" : 3, "buildId" : "D818A3C2CE10C9180464EE505794530C1A90AEF7" }, { "b" : "7F18C951D000", "path" : "/lib64/libnss3.so", "elfType" : 3, "buildId" : "54B32A2179988CAE4A11FA8F86AF1E9E2760073E" }, { "b" : "7F18C92EE000", "path" : "/lib64/libnssutil3.so", "elfType" : 3, "buildId" : "450373B22AF5668A9C0325FEFC918B396A66273C" }, { "b" : "7F18C90EA000", "path" : "/lib64/libplds4.so", "elfType" : 3, "buildId" : "A9EF38B34F69655160214C303487A6881992B229" }, { "b" : "7F18C8EE5000", "path" : "/lib64/libplc4.so", "elfType" : 3, "buildId" : "CD8713077A5983BFA6C267691EC82A077A46842C" }, { "b" : "7F18C8CA9000", "path" : "/lib64/libnspr4.so", "elfType" : 3, "buildId" : "05DADE6B01040412C2F7C18DE47056558821D028" }, { "b" : "7F18C8A82000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "8271DC15F869AE83F5A370302B4C3F0D9D6338E4" }, { "b" : "7F18C884B000", "path" : "/lib64/libcrypt.so.1", "elfType" : 3, "buildId" : "89E07B214D9DE630668A0E514B097098A90EA589" }, { "b" : "7F18C85E7000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "1E027913526FE9BC71D008691D3B7DBE88C880DE" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x556ee1ec4bf1]
mongod(+0x28BE20C) [0x556ee1ec420c]
mongod(+0x28BE296) [0x556ee1ec4296]
libpthread.so.0(+0x117E0) [0x7f18cc0477e0]
libc.so.6(gsignal+0x110) [0x7f18cbcbec20]
libc.so.6(abort+0x148) [0x7f18cbcc00c8]
mongod(_ZN5mongo42fassertFailedWithStatusNoTraceWithLocationEiRKNS_6StatusEPKcj+0x0) [0x556ee031aba1]
mongod(+0xA4DD4D) [0x556ee0053d4d]
mongod(_ZN5mongo21WiredTigerRecordStore26insertRecordsWithDocWriterEPNS_16OperationContextEPKPKNS_9DocWriterEPKNS_9TimestampEmPNS_8RecordIdE+0x1AC) [0x556ee0418f2c]
mongod(_ZN5mongo14CollectionImpl23insertDocumentsForOplogEPNS_16OperationContextEPKPKNS_9DocWriterEPNS_9TimestampEm+0x81) [0x556ee0be67e1]
mongod(_ZN5mongo4repl12_logOpsInnerEPNS_16OperationContextERKNS_15NamespaceStringEPKPKNS_9DocWriterEPNS_9TimestampEmPNS_10CollectionENS0_6OpTimeENS_6Date_tE+0xA3) [0x556ee0c7b613]
mongod(_ZN5mongo4repl5logOpEPNS_16OperationContextEPKcRKNS_15NamespaceStringEN5boost8optionalINS_4UUIDEEERKNS_7BSONObjEPSD_bNS_6Date_tERKNS_20OperationSessionInfoENS9_IiEERKNS0_9OplogLinkERKNS0_6OpTimeE+0x43C) [0x556ee0c7fdac]
mongod(+0x1461DBB) [0x556ee0a67dbb]
mongod(_ZN5mongo14OpObserverImpl8onUpdateEPNS_16OperationContextERKNS_20OplogUpdateEntryArgsE+0x7CC) [0x556ee0a6ddbc]
mongod(_ZN5mongo18OpObserverRegistry8onUpdateEPNS_16OperationContextERKNS_20OplogUpdateEntryArgsE+0x57) [0x556ee03a6107]
mongod(_ZN5mongo14CollectionImpl25updateDocumentWithDamagesEPNS_16OperationContextENS_8RecordIdERKNS_11SnapshottedINS_10RecordDataEEEPKcRKSt6vectorINS_11mutablebson11DamageEventESaISD_EEPNS_20CollectionUpdateArgsE+0x3BC) [0x556ee0becc8c]
mongod(_ZN5mongo11UpdateStage18transformAndUpdateERKNS_11SnapshottedINS_7BSONObjEEERNS_8RecordIdE+0x4ED) [0x556ee0dabb2d]
mongod(_ZN5mongo11UpdateStage6doWorkEPm+0x2F2) [0x556ee0dad212]
mongod(_ZN5mongo9PlanStage4workEPm+0x68) [0x556ee0d8b908]
mongod(_ZN5mongo16PlanExecutorImpl12_getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x230) [0x556ee0dd2e30]
mongod(_ZN5mongo16PlanExecutorImpl7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x4D) [0x556ee0dd358d]
mongod(+0x14928E7) [0x556ee0a988e7]
mongod(+0x14946FB) [0x556ee0a9a6fb]
mongod(+0x149596A) [0x556ee0a9b96a]
mongod(_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_3rpc21ReplyBuilderInterfaceE+0xAF) [0x556ee18b3cef]
mongod(+0x118B58A) [0x556ee079158a]
mongod(+0x118C875) [0x556ee0792875]
mongod(+0x118E8F4) [0x556ee07948f4]
mongod(_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE+0x41A) [0x556ee079568a]
mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x3C) [0x556ee078320c]
mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0xEC) [0x556ee078ef9c]
mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x17F) [0x556ee078c27f]
mongod(+0x1187F0C) [0x556ee078df0c]
mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE+0x182) [0x556ee15e9f82]
mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE+0x10D) [0x556ee078914d]
mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0x753) [0x556ee078a9c3]
mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x316) [0x556ee078b576]
mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0xDB) [0x556ee078c1db]
mongod(+0x1187F0C) [0x556ee078df0c]
mongod(+0x1FE43EB) [0x556ee15ea3eb]
mongod(+0x2639F35) [0x556ee1c3ff35]
mongod(+0x2639F94) [0x556ee1c3ff94]
libpthread.so.0(+0x740B) [0x7f18cc03d40b]
libc.so.6(clone+0x3F) [0x7f18cbd7809f]
----- END BACKTRACE -----
^C
[srikanta@ip-172-31-5-61 mongodb]$