Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-43954

Replica set sync failure with Invariant failure _inUnitOfWork() ActiveNotInUnitOfWork

    • Type: Icon: Bug Bug
    • Resolution: Duplicate
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 4.2.0
    • Component/s: None
    • None
    • ALL

      this happens when the replica server is creating indexes tried increasing ram and storage with no luck happens every time,
      MongoDB version 4.2
      Ubuntu 18.04 x64
      3 current replica secondary servers, and a primary

      Here is the stacktrace for same

       

      2019-10-11T05:10:07.312+0000 I INDEX [repl-writer-worker-11] build may temporarily use up to 500 megabytes of RAM
      2019-10-11T05:10:07.316+0000 I SHARDING [repl-writer-worker-4] Marking collection ads.supplyOffers as collection version: <unsharded>
      2
      
      2019-10-11T05:10:07.317+0000 F - [repl-writer-worker-4] Invariant failure _inUnitOfWork() ActiveNotInUnitOfWork src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp 318
      2019-10-11T05:10:07.317+0000 F - [repl-writer-worker-4]
      
      ***aborting after invariant() failure
      
      
      2019-10-11T05:10:07.340+0000 F - [repl-writer-worker-4] Got signal: 6 (Aborted).
       0x560b00af3b31 0x560b00af332e 0x560b00af33c6 0x7fd53363b890 0x7fd533276e97 0x7fd533278801 0x560aff02c4dd 0x560aff1182a9 0x560aff107a31 0x560aff11193b 0x560aff8e6c13 0x560aff8eafa6 0x560aff884c8a 0x560b0031d891 0x560b0031f055 0x560afeeb0a77 0x560aff3114e3 0x560aff311b6f 0x560aff42cad8 0x560aff42cf8c 0x560aff42b57e 0x560aff430899 0x560aff4314b3 0x560aff431aec 0x560aff9656b7 0x560aff966270 0x560aff967cb5 0x560b00c1981f 0x7fd5336306db 0x7fd53335988f
      ----- BEGIN BACKTRACE -----
      {"backtrace":[\{"b":"560AFE373000","o":"2780B31","s":"_ZN5mongo15printStackTraceERSo"},\{"b":"560AFE373000","o":"278032E"},\{"b":"560AFE373000","o":"27803C6"},\{"b":"7FD533629000","o":"12890"},\{"b":"7FD533238000","o":"3EE97","s":"gsignal"},\{"b":"7FD533238000","o":"40801","s":"abort"},\{"b":"560AFE373000","o":"CB94DD","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},\{"b":"560AFE373000","o":"DA52A9","s":"_ZN5mongo22WiredTigerRecoveryUnit14registerChangeEPNS_12RecoveryUnit6ChangeE"},\{"b":"560AFE373000","o":"D94A31","s":"_ZN5mongo21WiredTigerRecordStore17_increaseDataSizeEPNS_16OperationContextEl"},\{"b":"560AFE373000","o":"D9E93B","s":"_ZN5mongo21WiredTigerRecordStore12updateRecordEPNS_16OperationContextERKNS_8RecordIdEPKci"},\{"b":"560AFE373000","o":"1573C13","s":"_ZN5mongo18DurableCatalogImpl11putMetaDataEPNS_16OperationContextERKNS_15NamespaceStringERNS_26BSONCollectionCatalogEntry8MetaDataE"},\{"b":"560AFE373000","o":"1577FA6","s":"_ZN5mongo18DurableCatalogImpl18setIndexIsMultikeyEPNS_16OperationContextENS_15NamespaceStringENS_10StringDataERKSt6vectorISt3setImSt4lessImESaImEESaISA_EE"},\{"b":"560AFE373000","o":"1511C8A","s":"_ZN5mongo21IndexCatalogEntryImpl11setMultikeyEPNS_16OperationContextERKSt6vectorISt3setImSt4lessImESaImEESaIS8_EE"},\{"b":"560AFE373000","o":"1FAA891","s":"_ZN5mongo25AbstractIndexAccessMethod10insertKeysEPNS_16OperationContextERKSt6vectorINS_7BSONObjESaIS4_EES8_RKS3_ISt3setImSt4lessImESaImEESaISD_EERKNS_8RecordIdERKNS_19InsertDeleteOptionsEPNS_12InsertResultE"},\{"b":"560AFE373000","o":"1FAC055","s":"_ZN5mongo25AbstractIndexAccessMethod6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdERKNS_19InsertDeleteOptionsEPNS_12InsertResultE"},\{"b":"560AFE373000","o":"B3DA77"},\{"b":"560AFE373000","o":"F9E4E3","s":"_ZN5mongo4repl24CollectionBulkLoaderImpl25_addDocumentToIndexBlocksERKNS_7BSONObjERKNS_8RecordIdE"},\{"b":"560AFE373000","o":"F9EB6F","s":"_ZN5mongo4repl24CollectionBulkLoaderImpl15insertDocumentsEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS4_SaIS4_EEEESA_"},\{"b":"560AFE373000","o":"10B9AD8","s":"_ZN5mongo4repl16CollectionCloner24_insertDocumentsCallbackERKNS_8executor12TaskExecutor12CallbackArgsESt10shared_ptrINS0_23CallbackCompletionGuardINS_6StatusEEEE"},\{"b":"560AFE373000","o":"10B9F8C"},\{"b":"560AFE373000","o":"10B857E"},\{"b":"560AFE373000","o":"10BD899"},\{"b":"560AFE373000","o":"10BE4B3","s":"_ZN5mongo4repl10TaskRunner9_runTasksEv"},\{"b":"560AFE373000","o":"10BEAEC"},\{"b":"560AFE373000","o":"15F26B7","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockISt5mutexE"},\{"b":"560AFE373000","o":"15F3270","s":"_ZN5mongo10ThreadPool13_consumeTasksEv"},\{"b":"560AFE373000","o":"15F4CB5","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE"},\{"b":"560AFE373000","o":"28A681F"},\{"b":"7FD533629000","o":"76DB"},\{"b":"7FD533238000","o":"12188F","s":"clone"}],"processInfo":\{ "mongodbVersion" : "4.2.0", "gitVersion" : "a4b751dcf51dd249c5865812b390cfd1c0129c30", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.15.0-58-generic", "version" : "#64-Ubuntu SMP Tue Aug 6 11:12:41 UTC 2019", "machine" : "x86_64" }, "somap" : [ \{ "b" : "560AFE373000", "elfType" : 3, "buildId" : "D849CC14858F969B5888F892C9E16777E4C6FF32" }, \{ "b" : "7FFD703B0000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "D523EBC9181DE86DAB768A6167F8B401D3054550" }, \{ "b" : "7FD534B7D000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "1C6BC2C0699CE0F7E848CA0B267E0CF07553F6AB" }, \{ "b" : "7FD534962000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "390E9CC4C215314B6D8ADE6D6E28F8518418039C" }, \{ "b" : "7FD534497000", "path" : "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1", "elfType" : 3, "buildId" : "CB6876717C83B0CC01C3C919B9B6E86D8554F546" }, \{ "b" : "7FD53420A000", "path" : "/usr/lib/x86_64-linux-gnu/libssl.so.1.1", "elfType" : 3, "buildId" : "439A262CC0127BA401707DEC7A28884D617550E0" }, \{ "b" : "7FD534006000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "25AD56E902E23B490A9CCDB08A9744D89CB95BCC" }, \{ "b" : "7FD533DFE000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "9826FBDF57ED7D6965131074CB3C08B1009C1CD8" }, \{ "b" : "7FD533A60000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "A33761AB8FB485311B3C85BF4253099D7CABE653" }, \{ "b" : "7FD533848000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "41BDC55C07D5E5B1D8AB38E2C19B1F535855E084" }, \{ "b" : "7FD533629000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "28C6AADE70B2D40D1F0F3D0A1A0CAD1AB816448F" }, \{ "b" : "7FD533238000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B417C0BA7CC5CF06D1D1BED6652CEDB9253C60D0" }, \{ "b" : "7FD534DFC000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "64DF1B961228382FE18684249ED800AB1DCEAAD4" }, \{ "b" : "7FD533013000", "path" : "/usr/lib/x86_64-linux-gnu/libnghttp2.so.14", "elfType" : 3, "buildId" : "4F00E5207693FDC249DA42EC6472ACA6A7B929AE" }, \{ "b" : "7FD532DF6000", "path" : "/usr/lib/x86_64-linux-gnu/libidn2.so.0", "elfType" : 3, "buildId" : "BA5BF9A5C44F48C647E9D8270A5421AE81CCAD61" }, \{ "b" : "7FD532BDA000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "69465D8AA6B19086ABF2455A703F9168BF82A69F" }, \{ "b" : "7FD5329CC000", "path" : "/usr/lib/x86_64-linux-gnu/libpsl.so.5", "elfType" : 3, "buildId" : "CDAF1F1946846941F9D06414EC8C812D131A168E" }, \{ "b" : "7FD532781000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "00F419F64B0E70D8C5EEF7050369AA40B2A6E090" }, \{ "b" : "7FD53252F000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "70EEF126558D1559A0A4E334FB68E4E9AABE90CB" }, \{ "b" : "7FD532321000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "C14042EC7BD22B9A07D2C16563FE3C2606F52AB7" }, \{ "b" : "7FD532104000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "EF3E006DFE3132A41D4D4DC0E407D6EA658E11C4" }, \{ "b" : "7FD531D86000", "path" : "/usr/lib/x86_64-linux-gnu/libunistring.so.2", "elfType" : 3, "buildId" : "0E2784298E7D3F4D894FE130ACEFA77C3E624F72" }, \{ "b" : "7FD531A21000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "E5AE5C31F804BE96532D0DB2091F19E472F2D4A0" }, \{ "b" : "7FD5317ED000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "842BDF0B0EAAB82E19F1EABFC38769F4040FBE31" }, \{ "b" : "7FD5315B7000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "C20D4B3BA13FCDCC3BF6857689BA9FC70BE3F6A5" }, \{ "b" : "7FD531336000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "D40EA9B5EC5BC46799E4A412319617BD38BE9341" }, \{ "b" : "7FD531060000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "69FBCF425EE6DF03DE93B82FBC2FC33790E68A96" }, \{ "b" : "7FD530E2E000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "F400D5D643A7F9696DF0E6148FA99BEE6C1BDDF7" }, \{ "b" : "7FD530C2A000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "17107881DF65C66B4C6D38CAB37C285FA44663BD" }, \{ "b" : "7FD530A1F000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "D78D71E8E016A534281B25B97CD7E5E9DB5FE00A" }, \{ "b" : "7FD530804000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "ABB7E3F40302E6509DAD1F91DFB1F04B6A5FD072" }, \{ "b" : "7FD5305C3000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "A1A98DB481968073636BBAECB561A3EA8ED198AE" }, \{ "b" : "7FD530294000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "8DBD451EA5651283905E16FA7DFA9908688893A3" }, \{ "b" : "7FD530081000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "6036B89A3BB671B32E01464C0C82BFA016186352" }, \{ "b" : "7FD52FE7D000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "F463E107B099910463BC32E837C73D341A52C27B" }, \{ "b" : "7FD52FC74000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "C2376C5B831991591F1A67B976758185F86896D8" }, \{ "b" : "7FD52F9E7000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "69BDEE5FA0FEEDF317308BE850F78761861D520A" }, \{ "b" : "7FD52F745000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "315D74995AAA32DE4D15BA25F335066988B1B230" }, \{ "b" : "7FD52F50F000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "6673972A1C24A89EBAFBAE696188A4CB26C6DDEB" }, \{ "b" : "7FD52F2F9000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "430827C33259C12248CF44B91A9A9821114376F5" }, \{ "b" : "7FD52F0F1000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "3555B5F599C9787DFDDBF9E8DF6F706B9044D985" }, \{ "b" : "7FD52EEC8000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "93A0931B1C2818F0EA224CE6FE5E31E84A9B55BB" }, \{ "b" : "7FD52ECB9000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "669D4CCE42FA4382796EFFCF0C16F459F4382C4C" }, \{ "b" : "7FD52EA6F000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "4B80C543356EE0AF9039EFE7C9EA1CC1F74C426A" }, \{ "b" : "7FD52E766000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "BCE351987CF42B3D258B09F0CAC867758D935086" }, \{ "b" : "7FD52E52E000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "810686AF0D5FD350A4FB1CC4B5AFF44A05C102CB" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x560b00af3b31]
       mongod(+0x278032E) [0x560b00af332e]
       mongod(+0x27803C6) [0x560b00af33c6]
       libpthread.so.0(+0x12890) [0x7fd53363b890]
       libc.so.6(gsignal+0xC7) [0x7fd533276e97]
       libc.so.6(abort+0x141) [0x7fd533278801]
       mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0x0) [0x560aff02c4dd]
       mongod(_ZN5mongo22WiredTigerRecoveryUnit14registerChangeEPNS_12RecoveryUnit6ChangeE+0xB9) [0x560aff1182a9]
       mongod(_ZN5mongo21WiredTigerRecordStore17_increaseDataSizeEPNS_16OperationContextEl+0x81) [0x560aff107a31]
       mongod(_ZN5mongo21WiredTigerRecordStore12updateRecordEPNS_16OperationContextERKNS_8RecordIdEPKci+0x18B) [0x560aff11193b]
       mongod(_ZN5mongo18DurableCatalogImpl11putMetaDataEPNS_16OperationContextERKNS_15NamespaceStringERNS_26BSONCollectionCatalogEntry8MetaDataE+0x4E3) [0x560aff8e6c13]
       mongod(_ZN5mongo18DurableCatalogImpl18setIndexIsMultikeyEPNS_16OperationContextENS_15NamespaceStringENS_10StringDataERKSt6vectorISt3setImSt4lessImESaImEESaISA_EE+0x276) [0x560aff8eafa6]
       mongod(_ZN5mongo21IndexCatalogEntryImpl11setMultikeyEPNS_16OperationContextERKSt6vectorISt3setImSt4lessImESaImEESaIS8_EE+0x2EA) [0x560aff884c8a]
       mongod(_ZN5mongo25AbstractIndexAccessMethod10insertKeysEPNS_16OperationContextERKSt6vectorINS_7BSONObjESaIS4_EES8_RKS3_ISt3setImSt4lessImESaImEESaISD_EERKNS_8RecordIdERKNS_19InsertDeleteOptionsEPNS_12InsertResultE+0x3A1) [0x560b0031d891]
       mongod(_ZN5mongo25AbstractIndexAccessMethod6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdERKNS_19InsertDeleteOptionsEPNS_12InsertResultE+0x265) [0x560b0031f055]
       mongod(+0xB3DA77) [0x560afeeb0a77]
       mongod(_ZN5mongo4repl24CollectionBulkLoaderImpl25_addDocumentToIndexBlocksERKNS_7BSONObjERKNS_8RecordIdE+0x93) [0x560aff3114e3]
       mongod(_ZN5mongo4repl24CollectionBulkLoaderImpl15insertDocumentsEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS4_SaIS4_EEEESA_+0x55F) [0x560aff311b6f]
       mongod(_ZN5mongo4repl16CollectionCloner24_insertDocumentsCallbackERKNS_8executor12TaskExecutor12CallbackArgsESt10shared_ptrINS0_23CallbackCompletionGuardINS_6StatusEEEE+0x188) [0x560aff42cad8]
       mongod(+0x10B9F8C) [0x560aff42cf8c]
       mongod(+0x10B857E) [0x560aff42b57e]
       mongod(+0x10BD899) [0x560aff430899]
       mongod(_ZN5mongo4repl10TaskRunner9_runTasksEv+0xB3) [0x560aff4314b3]
       mongod(+0x10BEAEC) [0x560aff431aec]
       mongod(_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockISt5mutexE+0xF7) [0x560aff9656b7]
       mongod(_ZN5mongo10ThreadPool13_consumeTasksEv+0xA0) [0x560aff966270]
       mongod(_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE+0x95) [0x560aff967cb5]
       mongod(+0x28A681F) [0x560b00c1981f]
       libpthread.so.0(+0x76DB) [0x7fd5336306db]
       libc.so.6(clone+0x3F) [0x7fd53335988f]
      ----- END BACKTRACE -----
      

            Assignee:
            daniel.hatcher@mongodb.com Danny Hatcher (Inactive)
            Reporter:
            phpmalik@gmail.com Puspender Malik
            Votes:
            0 Vote for this issue
            Watchers:
            7 Start watching this issue

              Created:
              Updated:
              Resolved: