Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-53537

MongoDB crashed after state change

    • Type: Icon: Bug Bug
    • Resolution: Duplicate
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 4.4.2
    • Component/s: None
    • None
    • ALL

      Hello,

      I need help as I don't understand why mongod crashed. Mongod crashed after state change from Secondary to Primary.

      What could be the reason for the crash?

      Log

      {"t":{"$date":"2020-12-28T00:33:36.420+03:00"},"s":"I",  "c":"ELECTION", "id":21450,   "ctx":"ReplCoord-4690","msg":"Election succeeded, assuming primary role","attr":{"term":182}}
      {"t":{"$date":"2020-12-28T00:33:36.421+03:00"},"s":"I",  "c":"REPL",     "id":21358,   "ctx":"ReplCoord-4690","msg":"Replica set state transition","attr":{"newState":"PRIMARY","oldState":"SECONDARY"}}
      {"t":{"$date":"2020-12-28T00:33:36.441+03:00"},"s":"I",  "c":"REPL",     "id":21106,   "ctx":"ReplCoord-4690","msg":"Resetting sync source to empty","attr":{"previousSyncSource":"mongo46:27017"}}
      {"t":{"$date":"2020-12-28T00:33:36.442+03:00"},"s":"I",  "c":"CONNPOOL", "id":22566,   "ctx":"ReplNetwork","msg":"Ending connection due to bad connection status","attr":{"hostAndPort":"mongo46:27017","error":"CallbackCanceled: Callback was canceled","numOpenConns":2}}
      {"t":{"$date":"2020-12-28T00:33:36.442+03:00"},"s":"I",  "c":"REPL",     "id":21359,   "ctx":"ReplCoord-4690","msg":"Entering primary catch-up mode"}
      {"t":{"$date":"2020-12-28T00:33:36.463+03:00"},"s":"F",  "c":"-",        "id":23079,   "ctx":"TopologyVersionObserver","msg":"Invariant failure","attr":{"expr":"opCtx != nullptr && _opCtx == nullptr","file":"src/mongo/db/client.cpp","line":126}}
      {"t":{"$date":"2020-12-28T00:33:36.466+03:00"},"s":"F",  "c":"-",        "id":23080,   "ctx":"TopologyVersionObserver","msg":"\n\n***aborting after invariant() failure\n\n"}
      {"t":{"$date":"2020-12-28T00:33:36.467+03:00"},"s":"F",  "c":"CONTROL",  "id":4757800, "ctx":"TopologyVersionObserver","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
      {"t":{"$date":"2020-12-28T00:33:36.487+03:00"},"s":"I",  "c":"COMMAND",  "id":51803,   "ctx":"conn28221","msg":"Slow query","attr":{"type":"command","ns":"irkkt.STATS.med_care","command":{"aggregate":"STATS.med_care","pipeline":[{"$match":{"$and":[{"federal_subject_code":{"$in":["19"]}},{"op_date":{"$gte":{"$date":"2020-11-01T00:00:00.000Z"},"$lt":{"$date":"2020-11-30T23:59:59.000Z"}}}]}},{"$group":{"_id":null,"sum":{"$sum":"$kiz_count"}}}],"cursor":{},"lsid":{"id":{"$uuid":"82fd217c-2909-4f04-875d-295d32e75068"}},"$clusterTime":{"clusterTime":{"$timestamp":{"t":1609104803,"i":50}},"signature":{"hash":{"$binary":{"base64":"AAAAAAAAAAAAAAAAAAAAAAAAAAA=","subType":"0"}},"keyId":0}},"$db":"irkkt","$readPreference":{"mode":"secondary"}},"planSummary":"IXSCAN { op_date: -1, federal_subject_code: 1 }","keysExamined":277077,"docsExamined":1618,"cursorExhausted":true,"numYields":277,"nreturned":1,"queryHash":"AB897A3D","planCacheKey":"FA757DED","reslen":256,"locks":{"ReplicationStateTransition":{"acquireCount":{"w":279}},"Global":{"acquireCount":{"r":279}},"Database":{"acquireCount":{"r":279}},"Collection":{"acquireCount":{"r":279}},"Mutex":{"acquireCount":{"r":2}}},"storage":{"data":{"bytesRead":17324253,"timeReadingMicros":22990}},"protocol":"op_msg","durationMillis":511}}
      {"t":{"$date":"2020-12-28T00:33:36.514+03:00"},"s":"I",  "c":"NETWORK",  "id":4333213, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM Topology Change","attr":{"replicaSet":"testing","newTopologyDescription":"{ id: \"7714ed9e-5260-4338-87d8-c2f62995024e\", topologyType: \"ReplicaSetWithPrimary\", servers: { mongo44:27017: { address: \"mongo44:27017\", topologyVersion: { processId: ObjectId('5fde58d937bb66a7a0a75c61'), counter: 5 }, roundTripTime: 841, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo44:27017\", setName: \"testing\", setVersion: 86588, primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104812544), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo45:27017: { address: \"mongo45:27017\", topologyVersion: { processId: ObjectId('5fde5a21f8d2dbb73e37809d'), counter: 8 }, roundTripTime: 481, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo45:27017\", setName: \"testing\", setVersion: 86588, primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104809561), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo46:27017: { address: \"mongo46:27017\", topologyVersion: { processId: ObjectId('5fde5af01fe8658300d3f47f'), counter: 7 }, roundTripTime: 490, lastWriteDate: new Date(1609104799000), opTime: { ts: Timestamp(1609104799, 353), t: 181 }, type: \"RSPrimary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo46:27017\", setName: \"testing\", setVersion: 86588, electionId: ObjectId('7fffffff00000000000000b5'), primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104799952), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo55:27017: { address: \"mongo55:27017\", topologyVersion: { processId: ObjectId('5fde5985b762cb0d7b23e740'), counter: 6 }, roundTripTime: 353, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo55:27017\", setName: \"testing\", setVersion: 86588, electionId: ObjectId('7fffffff00000000000000b6'), primary: \"mongo55:27017\", lastUpdateTime: new Date(1609104816513), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} } }, logicalSessionTimeoutMinutes: 30, setName: \"testing\", compatible: true, maxSetVersion: 86588, maxElectionId: ObjectId('7fffffff00000000000000b5') }","previousTopologyDescription":"{ id: \"7714ed9e-5260-4338-87d8-c2f62995024e\", topologyType: \"ReplicaSetWithPrimary\", servers: { mongo44:27017: { address: \"mongo44:27017\", topologyVersion: { processId: ObjectId('5fde58d937bb66a7a0a75c61'), counter: 5 }, roundTripTime: 841, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo44:27017\", setName: \"testing\", setVersion: 86588, primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104812544), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo45:27017: { address: \"mongo45:27017\", topologyVersion: { processId: ObjectId('5fde5a21f8d2dbb73e37809d'), counter: 8 }, roundTripTime: 481, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo45:27017\", setName: \"testing\", setVersion: 86588, primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104809561), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo46:27017: { address: \"mongo46:27017\", topologyVersion: { processId: ObjectId('5fde5af01fe8658300d3f47f'), counter: 7 }, roundTripTime: 490, lastWriteDate: new Date(1609104799000), opTime: { ts: Timestamp(1609104799, 353), t: 181 }, type: \"RSPrimary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo46:27017\", setName: \"testing\", setVersion: 86588, electionId: ObjectId('7fffffff00000000000000b5'), primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104799952), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} }, mongo55:27017: { address: \"mongo55:27017\", topologyVersion: { processId: ObjectId('5fde5985b762cb0d7b23e740'), counter: 5 }, roundTripTime: 353, lastWriteDate: new Date(1609104803000), opTime: { ts: Timestamp(1609104803, 50), t: 181 }, type: \"RSSecondary\", minWireVersion: 9, maxWireVersion: 9, me: \"mongo55:27017\", setName: \"testing\", setVersion: 86588, primary: \"mongo46:27017\", lastUpdateTime: new Date(1609104809783), logicalSessionTimeoutMinutes: 30, hosts: { 0: \"mongo44:27017\", 1: \"mongo45:27017\", 2: \"mongo46:27017\", 3: \"mongo55:27017\" }, arbiters: {}, passives: {} } }, logicalSessionTimeoutMinutes: 30, setName: \"testing\", compatible: true, maxSetVersion: 86588, maxElectionId: ObjectId('7fffffff00000000000000b5') }"}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31431,   "ctx":"TopologyVersionObserver","msg":"BACKTRACE: {bt}","attr":{"bt":{"backtrace":[{"a":"56047F7DA2C1","b":"56047CAF6000","o":"2CE42C1","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1E1"},{"a":"56047F7DB8F9","b":"56047CAF6000","o":"2CE58F9","s":"_ZN5mongo15printStackTraceEv","s+":"29"},{"a":"56047F7D9156","b":"56047CAF6000","o":"2CE3156","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"},{"a":"7F65EDB41630","b":"7F65EDB32000","o":"F630","s":"_L_unlock_13","s+":"34"},{"a":"7F65ED79A387","b":"7F65ED764000","o":"36387","s":"gsignal","s+":"37"},{"a":"7F65ED79BA78","b":"7F65ED764000","o":"37A78","s":"abort","s+":"148"},{"a":"56047DA0B0E9","b":"56047CAF6000","o":"F150E9","s":"_ZN5mongo15invariantFailedEPKcS1_j","s+":"12C"},{"a":"56047D9DE7FA","b":"56047CAF6000","o":"EE87FA","s":"_ZN5mongo6Client19setOperationContextEPNS_16OperationContextE.cold.135","s+":"18"},{"a":"56047F69B819","b":"56047CAF6000","o":"2BA5819","s":"_ZN5mongo14ServiceContext20makeOperationContextEPNS_6ClientE","s+":"129"},{"a":"56047F6902C7","b":"56047CAF6000","o":"2B9A2C7","s":"_ZN5mongo6Client20makeOperationContextEv","s+":"27"},{"a":"56047DF113A1","b":"56047CAF6000","o":"141B3A1","s":"_ZN5mongo4repl23TopologyVersionObserver17_workerThreadBodyEv","s+":"251"},{"a":"56047DF11A4C","b":"56047CAF6000","o":"141BA4C","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_4repl23TopologyVersionObserver4initEPNS3_14ServiceContextEPNS7_22ReplicationCoordinatorEEUlvE0_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"5C"},{"a":"56047F97FFAF","b":"56047CAF6000","o":"2E89FAF","s":"execute_native_thread_routine","s+":"F"},{"a":"7F65EDB39EA5","b":"7F65EDB32000","o":"7EA5","s":"start_thread","s+":"C5"},{"a":"7F65ED86296D","b":"7F65ED764000","o":"FE96D","s":"clone","s+":"6D"}],"processInfo":{"mongodbVersion":"4.4.2","gitVersion":"15e73dc5738d2278b688f8929aee605fe4279b0e","compiledModules":[],"uname":{"sysname":"Linux","release":"3.10.0-1160.6.1.el7.x86_64","version":"#1 SMP Tue Nov 17 13:59:11 UTC 2020","machine":"x86_64"},"somap":[{"b":"56047CAF6000","elfType":3,"buildId":"80A90728DC9FDCCF9246FBEBAEB8F415D7FA8827"},{"b":"7F65EDB32000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"2B482B3BAE79DEF4E5BC9791BC6BBDAE0E93E359"},{"b":"7F65ED764000","path":"/lib64/libc.so.6","elfType":3,"buildId":"F9FAFDE281E0E0E2AF45911AD0FA115B64C2CEA8"}]}}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"56047F7DA2C1","b":"56047CAF6000","o":"2CE42C1","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1E1"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"56047F7DB8F9","b":"56047CAF6000","o":"2CE58F9","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"56047F7D9156","b":"56047CAF6000","o":"2CE3156","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F65EDB41630","b":"7F65EDB32000","o":"F630","s":"_L_unlock_13","s+":"34"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F65ED79A387","b":"7F65ED764000","o":"36387","s":"gsignal","s+":"37"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F65ED79BA78","b":"7F65ED764000","o":"37A78","s":"abort","s+":"148"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"56047DA0B0E9","b":"56047CAF6000","o":"F150E9","s":"_ZN5mongo15invariantFailedEPKcS1_j","s+":"12C"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"56047D9DE7FA","b":"56047CAF6000","o":"EE87FA","s":"_ZN5mongo6Client19setOperationContextEPNS_16OperationContextE.cold.135","s+":"18"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"56047F69B819","b":"56047CAF6000","o":"2BA5819","s":"_ZN5mongo14ServiceContext20makeOperationContextEPNS_6ClientE","s+":"129"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"56047F6902C7","b":"56047CAF6000","o":"2B9A2C7","s":"_ZN5mongo6Client20makeOperationContextEv","s+":"27"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"56047DF113A1","b":"56047CAF6000","o":"141B3A1","s":"_ZN5mongo4repl23TopologyVersionObserver17_workerThreadBodyEv","s+":"251"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"56047DF11A4C","b":"56047CAF6000","o":"141BA4C","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_4repl23TopologyVersionObserver4initEPNS3_14ServiceContextEPNS7_22ReplicationCoordinatorEEUlvE0_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"5C"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"56047F97FFAF","b":"56047CAF6000","o":"2E89FAF","s":"execute_native_thread_routine","s+":"F"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F65EDB39EA5","b":"7F65EDB32000","o":"7EA5","s":"start_thread","s+":"C5"}}}
      {"t":{"$date":"2020-12-28T00:33:36.820+03:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"TopologyVersionObserver","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F65ED86296D","b":"7F65ED764000","o":"FE96D","s":"clone","s+":"6D"}}}
      

       

            Assignee:
            dmitry.agranat@mongodb.com Dmitry Agranat
            Reporter:
            kkrasnov@gmail.com Konstantin Krasnov
            Votes:
            0 Vote for this issue
            Watchers:
            11 Start watching this issue

              Created:
              Updated:
              Resolved: