-
Type: Bug
-
Resolution: Duplicate
-
Priority: Major - P3
-
None
-
Affects Version/s: 4.4.2
-
Component/s: None
-
None
-
ALL
-
(copied to CRM)
Hello,
I need help as I don't understand why mongod crashed. Mongod crashed after state change from Secondary to Primary.
What could be the reason for the crash?
Log
{"t":{"$date":"2020-12-28T00:33:36.420+03:00"},"s":"I", "c":"ELECTION", "id":21450, "ctx":"ReplCoord-4690","msg":"Election succeeded, assuming primary role","attr":{"term":182}} {"t":{"$date":"2020-12-28T00:33:36.421+03:00"},"s":"I", "c":"REPL", "id":21358, "ctx":"ReplCoord-4690","msg":"Replica set state transition","attr":{"newState":"PRIMARY","oldState":"SECONDARY"}} {"t":{"$date":"2020-12-28T00:33:36.441+03:00"},"s":"I", "c":"REPL", "id":21106, "ctx":"ReplCoord-4690","msg":"Resetting sync source to empty","attr":{"previousSyncSource":"mongo46:27017"}} {"t":{"$date":"2020-12-28T00:33:36.442+03:00"},"s":"I", "c":"CONNPOOL", "id":22566, "ctx":"ReplNetwork","msg":"Ending connection due to bad connection status","attr":{"hostAndPort":"mongo46:27017","error":"CallbackCanceled: Callback was canceled","numOpenConns":2}} {"t":{"$date":"2020-12-28T00:33:36.442+03:00"},"s":"I", "c":"REPL", "id":21359, "ctx":"ReplCoord-4690","msg":"Entering primary catch-up mode"} {"t":{"$date":"2020-12-28T00:33:36.463+03:00"},"s":"F", "c":"-", "id":23079, "ctx":"TopologyVersionObserver","msg":"Invariant failure","attr":{"expr":"opCtx != nullptr && _opCtx == nullptr","file":"src/mongo/db/client.cpp","line":126}} {"t":{"$date":"2020-12-28T00:33:36.466+03:00"},"s":"F", "c":"-", "id":23080, "ctx":"TopologyVersionObserver","msg":"\n\n***aborting after invariant() failure\n\n"} {"t":{"$date":"2020-12-28T00:33:36.467+03:00"},"s":"F", "c":"CONTROL", "id":4757800, "ctx":"TopologyVersionObserver","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}} {"t":{"$date":"2020-12-28T00:33:36.487+03:00"},"s":"I", "c":"COMMAND", "id":51803, "ctx":"conn28221","msg":"Slow query","attr":{"type":"command","ns":"irkkt.STATS.med_care","command":{"aggregate":"STATS.med_care","pipeline":[{"$match":{"$and":[{"federal_subject_code":{"$in":["19"]}},{"op_date":{"$gte":{"$date":"2020-11-01T00:00:00.000Z"},"$lt":{"$date":"2020-11-30T23:59:59.000Z"}}}]}},{"$group":{"_id":null,"sum":{"$sum":"$kiz_count"}}}],"cursor":{},"lsid":{"id":{"$uuid":"82fd217c-2909-4f04-875d-295d32e75068"}},"$clusterTime":{"clusterTime":{"$timestamp":{"t":1609104803,"i":50}},"signature":{"hash":{"$binary":{"base64":"AAAAAAAAAAAAAAAAAAAAAAAAAAA=","subType":"0"}},"keyId":0}},"$db":"irkkt","$readPreference":{"mode":"secondary"}},"planSummary":"IXSCAN { op_date: -1, federal_subject_code: 1 }","keysExamined":277077,"docsExamined":1618,"cursorExhausted":true,"numYields":277,"nreturned":1,"queryHash":"AB897A3D","planCacheKey":"FA757DED","reslen":256,"locks":{"ReplicationStateTransition":{"acquireCount":{"w":279}},"Global":{"acquireCount":{"r":279}},"Database":{"acquireCount":{"r":279}},"Collection":{"acquireCount":{"r":279}},"Mutex":{"acquireCount":{"r":2}}},"storage":{"data":{"bytesRead":17324253,"timeReadingMicros":22990}},"protocol":"op_msg","durationMillis":511}} {"t":{"$date":"2020-12-28T00:33:36.514+03:00"},"s":"I", "c":"NETWORK", "id":4333213, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM Topology Change","attr":{"replicaSet":"testing","newTopologyDescription":"{ id: \"7714ed9e-5260-4338-87d8-c2f62995024e\", topologyType: \"ReplicaSetWithPrimary\", servers: { mongo44:27017: { address: \"mongo44:27017\", topologyVersion: { processId: ObjectId('5fde58d937bb66a7a0a75c61'), counter: 5 }, roundTripTime: 841, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo44:27017\", setName: \"testing\", setVersion: 86588, primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104812544), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo45:27017: { address: \"mongo45:27017\", topologyVersion: { processId: ObjectId('5fde5a21f8d2dbb73e37809d'), counter: 8 }, roundTripTime: 481, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo45:27017\", setName: \"testing\", setVersion: 86588, primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104809561), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo46:27017: { address: \"mongo46:27017\", topologyVersion: { processId: ObjectId('5fde5af01fe8658300d3f47f'), counter: 7 }, roundTripTime: 490, lastWriteDate: new Date(1609104799000), opTime: { ts: Timestamp(1609104799, 353), t: 181 }, type: \"RSPrimary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo46:27017\", setName: \"testing\", setVersion: 86588, electionId: ObjectId('7fffffff00000000000000b5'), primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104799952), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo55:27017: { address: \"mongo55:27017\", topologyVersion: { processId: ObjectId('5fde5985b762cb0d7b23e740'), counter: 6 }, roundTripTime: 353, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo55:27017\", setName: \"testing\", setVersion: 86588, electionId: ObjectId('7fffffff00000000000000b6'), primary: \"mongo55:27017\", lastUpdateTime: new Date(1609104816513), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} } }, logicalSessionTimeoutMinutes: 30, setName: \"testing\", compatible: true, maxSetVersion: 86588, maxElectionId: ObjectId('7fffffff00000000000000b5') }","previousTopologyDescription":"{ id: \"7714ed9e-5260-4338-87d8-c2f62995024e\", topologyType: \"ReplicaSetWithPrimary\", servers: { mongo44:27017: { address: \"mongo44:27017\", topologyVersion: { processId: ObjectId('5fde58d937bb66a7a0a75c61'), counter: 5 }, roundTripTime: 841, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo44:27017\", setName: \"testing\", setVersion: 86588, primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104812544), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo45:27017: { address: \"mongo45:27017\", topologyVersion: { processId: ObjectId('5fde5a21f8d2dbb73e37809d'), counter: 8 }, roundTripTime: 481, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo45:27017\", setName: \"testing\", setVersion: 86588, primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104809561), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo46:27017: { address: \"mongo46:27017\", topologyVersion: { processId: ObjectId('5fde5af01fe8658300d3f47f'), counter: 7 }, roundTripTime: 490, lastWriteDate: new Date(1609104799000), opTime: { ts: Timestamp(1609104799, 353), t: 181 }, type: \"RSPrimary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo46:27017\", setName: \"testing\", setVersion: 86588, electionId: ObjectId('7fffffff00000000000000b5'), primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104799952), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo55:27017: { address: \"mongo55:27017\", topologyVersion: { processId: ObjectId('5fde5985b762cb0d7b23e740'), counter: 5 }, roundTripTime: 353, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo55:27017\", setName: \"testing\", setVersion: 86588, primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104809783), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} } }, logicalSessionTimeoutMinutes: 30, setName: \"testing\", compatible: true, maxSetVersion: 86588, maxElectionId: ObjectId('7fffffff00000000000000b5') }"}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31431, "ctx":"TopologyVersionObserver","msg":"BACKTRACE: {bt}","attr":{"bt":{"backtrace":[{"a":"56047F7DA2C1","b":"56047CAF6000","o":"2CE42C1","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1E1"},{"a":"56047F7DB8F9","b":"56047CAF6000","o":"2CE58F9","s":"_ZN5mongo15printStackTraceEv","s+":"29"},{"a":"56047F7D9156","b":"56047CAF6000","o":"2CE3156","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"},{"a":"7F65EDB41630","b":"7F65EDB32000","o":"F630","s":"_L_unlock_13","s+":"34"},{"a":"7F65ED79A387","b":"7F65ED764000","o":"36387","s":"gsignal","s+":"37"},{"a":"7F65ED79BA78","b":"7F65ED764000","o":"37A78","s":"abort","s+":"148"},{"a":"56047DA0B0E9","b":"56047CAF6000","o":"F150E9","s":"_ZN5mongo15invariantFailedEPKcS1_j","s+":"12C"},{"a":"56047D9DE7FA","b":"56047CAF6000","o":"EE87FA","s":"_ZN5mongo6Client19setOperationContextEPNS_16OperationContextE.cold.135","s+":"18"},{"a":"56047F69B819","b":"56047CAF6000","o":"2BA5819","s":"_ZN5mongo14ServiceContext20makeOperationContextEPNS_6ClientE","s+":"129"},{"a":"56047F6902C7","b":"56047CAF6000","o":"2B9A2C7","s":"_ZN5mongo6Client20makeOperationContextEv","s+":"27"},{"a":"56047DF113A1","b":"56047CAF6000","o":"141B3A1","s":"_ZN5mongo4repl23TopologyVersionObserver17_workerThreadBodyEv","s+":"251"},{"a":"56047DF11A4C","b":"56047CAF6000","o":"141BA4C","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_4repl23TopologyVersionObserver4initEPNS3_14ServiceContextEPNS7_22ReplicationCoordinatorEEUlvE0_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"5C"},{"a":"56047F97FFAF","b":"56047CAF6000","o":"2E89FAF","s":"execute_native_thread_routine","s+":"F"},{"a":"7F65EDB39EA5","b":"7F65EDB32000","o":"7EA5","s":"start_thread","s+":"C5"},{"a":"7F65ED86296D","b":"7F65ED764000","o":"FE96D","s":"clone","s+":"6D"}],"processInfo":{"mongodbVersion":"4.4.2","gitVersion":"15e73dc5738d2278b688f8929aee605fe4279b0e","compiledModules":[],"uname":{"sysname":"Linux","release":"3.10.0-1160.6.1.el7.x86_64","version":"#1 SMP Tue Nov 17 13:59:11 UTC 2020","machine":"x86_64"},"somap":[{"b":"56047CAF6000","elfType":3,"buildId":"80A90728DC9FDCCF9246FBEBAEB8F415D7FA8827"},{"b":"7F65EDB32000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"2B482B3BAE79DEF4E5BC9791BC6BBDAE0E93E359"},{"b":"7F65ED764000","path":"/lib64/libc.so.6","elfType":3,"buildId":"F9FAFDE281E0E0E2AF45911AD0FA115B64C2CEA8"}]}}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"56047F7DA2C1","b":"56047CAF6000","o":"2CE42C1","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1E1"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"56047F7DB8F9","b":"56047CAF6000","o":"2CE58F9","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"56047F7D9156","b":"56047CAF6000","o":"2CE3156","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"7F65EDB41630","b":"7F65EDB32000","o":"F630","s":"_L_unlock_13","s+":"34"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"7F65ED79A387","b":"7F65ED764000","o":"36387","s":"gsignal","s+":"37"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"7F65ED79BA78","b":"7F65ED764000","o":"37A78","s":"abort","s+":"148"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"56047DA0B0E9","b":"56047CAF6000","o":"F150E9","s":"_ZN5mongo15invariantFailedEPKcS1_j","s+":"12C"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"56047D9DE7FA","b":"56047CAF6000","o":"EE87FA","s":"_ZN5mongo6Client19setOperationContextEPNS_16OperationContextE.cold.135","s+":"18"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"56047F69B819","b":"56047CAF6000","o":"2BA5819","s":"_ZN5mongo14ServiceContext20makeOperationContextEPNS_6ClientE","s+":"129"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"56047F6902C7","b":"56047CAF6000","o":"2B9A2C7","s":"_ZN5mongo6Client20makeOperationContextEv","s+":"27"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"56047DF113A1","b":"56047CAF6000","o":"141B3A1","s":"_ZN5mongo4repl23TopologyVersionObserver17_workerThreadBodyEv","s+":"251"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"56047DF11A4C","b":"56047CAF6000","o":"141BA4C","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_4repl23TopologyVersionObserver4initEPNS3_14ServiceContextEPNS7_22ReplicationCoordinatorEEUlvE0_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"5C"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"56047F97FFAF","b":"56047CAF6000","o":"2E89FAF","s":"execute_native_thread_routine","s+":"F"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"7F65EDB39EA5","b":"7F65EDB32000","o":"7EA5","s":"start_thread","s+":"C5"}}} {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"TopologyVersionObserver","msg":" Frame: {frame}","attr":{"frame":{"a":"7F65ED86296D","b":"7F65ED764000","o":"FE96D","s":"clone","s+":"6D"}}}
- is duplicated by
-
SERVER-52735 mongodb crash with "Invariant failure" opCtx != nullptr && _opCtx == nullptr
- Closed
- related to
-
SERVER-52735 mongodb crash with "Invariant failure" opCtx != nullptr && _opCtx == nullptr
- Closed