Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-23459

Mongos 3.2.0 crash after this error --Invariant failure iter != pool.end()

    • Type: Icon: Bug Bug
    • Resolution: Duplicate
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 3.2.0
    • Component/s: Networking
    • None
    • Environment:
      centos 6.5
    • ALL

      2016-03-29T11:33:12.452+0800 I SHARDING [Balancer] distributed lock with ts: 56f9f77874649daca319c905' unlocked.
      2016-03-29T11:33:22.587+0800 I SHARDING [Balancer] distributed lock 'balancer' acquired for 'doing balance round', ts : 56f9f78274649daca319c907
      2016-03-29T11:33:22.618+0800 I SHARDING [Balancer] about to log metadata event into actionlog: { _id: "yw28-2016-03-29T11:33:22.618+0800-56f9f78274649daca319c908", server: "yw28", clientAddr: "", time: new Date(1459222402618), what: "balancer.round", ns: "", details: { executionTimeMillis: 158, errorOccured: false, candidateChunks: 0, chunksMoved: 0 } }
      2016-03-29T11:33:22.653+0800 I SHARDING [Balancer] distributed lock with ts: 56f9f78274649daca319c907' unlocked.
      2016-03-29T11:33:25.551+0800 I -        [NetworkInterfaceASIO-TaskExecutorPool-3-0] Invariant failure iter != pool.end() src/mongo/executor/connection_pool.cpp 572
      2016-03-29T11:33:25.551+0800 I -        [NetworkInterfaceASIO-TaskExecutorPool-3-0]
      
      ***aborting after invariant() failure
      
      
      2016-03-29T11:33:25.585+0800 F -        [NetworkInterfaceASIO-TaskExecutorPool-3-0] Got signal: 6 (Aborted).
      
       0xc3ea72 0xc3d9b9 0xc3e1c2 0x314560f790 0x3145232625 0x3145233e05 0xbc506b 0x9d3e50 0x9d58b1 0x9d8c7c 0x9d97f7 0x9d9f61 0xc5b2a1 0xc5b4c1 0xc5f65f 0x9e31e5 0xe15d50 0x3145607a51 0x31452e893d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"83EA72"},{"b":"400000","o":"83D9B9"},{"b":"400000","o":"83E1C2"},{"b":"3145600000","o":"F790"},{"b":"3145200000","o":"32625"},{"b":"3145200000","o":"33E05"},{"b":"400000","o":"7C506B"},{"b":"400000","o":"5D3E50"},{"b":"400000","o":"5D58B1"},{"b":"400000","o":"5D8C7C"},{"b":"400000","o":"5D97F7"},{"b":"400000","o":"5D9F61"},{"b":"400000","o":"85B2A1"},{"b":"400000","o":"85B4C1"},{"b":"400000","o":"85F65F"},{"b":"400000","o":"5E31E5"},{"b":"400000","o":"A15D50"},{"b":"3145600000","o":"7A51"},{"b":"3145200000","o":"E893D"}],"processInfo":{ "mongodbVersion" : "3.2.0", "gitVersion" : "45d947729a0315accb6d4f15a6b06be6d9c19fe7", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "2.6.32-573.3.1.el6.x86_64", "version" : "#1 SMP Thu Aug 13 22:55:16 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "BB92F8B97A2EB49533F22BDB14278DA703038254" }, { "b" : "7FFF8A22F000", "elfType" : 3, "buildId" : "77385010026236DADA9BFEADC4717B7518E8BB29" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "6AC11C021C127E84A2724F3A750461A2183B1069" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "78A423B7DD3C00F7A77027D8D764BB432B57F1E7" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "58C5A5FF5C82D7BE3113BE36DD87C7004E3C4DB1" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "B5AE05CEDC0CE917F50A3A468CFA2ACD8592E8F6" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "C03877A9EE01DDC572E2B0F55F64C757773CF8D6" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "989FE3A42CA8CEBDCC185A743896F23A0CF537ED" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9350579A4970FA47F3144AD8F40B183B0954497D" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "D467973C46E563CDCF64B5F12B2D6A50C7A25BA1" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "87A19A61DC011579F3E25DE3CA9778C6FD9E4547" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "04202A4A8BE624D2193E812A25589E2DD02D5B5C" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "441FA45097A11508E50D55A3D1FF169BF2BE7C62" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "F62622218875795666E08B92D176A50791183EEC" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "B8DEDADC140347276164C729418C7A37B7224135" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "4BDFC7A19C1F328EB4FCFBCE7A1E27606928610D" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F704FA7D21D05EF31E90FB4890FCA7F3D91DA138" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
       mongos(_ZN5mongo15printStackTraceERSo+0x32) [0xc3ea72]
       mongos(+0x83D9B9) [0xc3d9b9]
       mongos(+0x83E1C2) [0xc3e1c2]
       libpthread.so.0(+0xF790) [0x314560f790]
       libc.so.6(gsignal+0x35) [0x3145232625]
       libc.so.6(abort+0x175) [0x3145233e05]
       mongos(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0xbc506b]
       mongos(_ZN5mongo8executor14ConnectionPool12SpecificPool22takeFromProcessingPoolEPNS1_19ConnectionInterfaceE+0x0) [0x9d3e50]
       mongos(+0x5D58B1) [0x9d58b1]
       mongos(+0x5D8C7C) [0x9d8c7c]
       mongos(+0x5D97F7) [0x9d97f7]
       mongos(+0x5D9F61) [0x9d9f61]
       mongos(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code+0x2F1) [0xc5b2a1]
       mongos(_ZN4asio6detail9scheduler3runERSt10error_code+0xC1) [0xc5b4c1]
       mongos(_ZN4asio10io_service3runEv+0x2F) [0xc5f65f]
       mongos(+0x5E31E5) [0x9e31e5]
       mongos(+0xA15D50) [0xe15d50]
       libpthread.so.0(+0x7A51) [0x3145607a51]
       libc.so.6(clone+0x6D) [0x31452e893d]
      -----  END BACKTRACE  -----
      

            Assignee:
            kelsey.schubert@mongodb.com Kelsey Schubert
            Reporter:
            panjy5566 iori.pan
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Created:
              Updated:
              Resolved: