-
Type: Bug
-
Resolution: Fixed
-
Priority: Critical - P2
-
Affects Version/s: 4.2.0
-
Component/s: Upgrade/Downgrade
-
None
-
Fully Compatible
-
ALL
-
-
Repl 2019-09-09, Repl 2019-09-23, Repl 2019-10-07
Here's the mongo log upon failure:
2019-08-21T19:40:51.308+0700 I SHARDING [conn5] Marking collection traveliko-analytics.bookingAnalyticsYearByCountryView as collection version: <unsharded>
2019-08-21T19:40:51.308+0700 I SHARDING [conn5] Marking collection traveliko-analytics.roomCalendarRateRequestView as collection version: <unsharded>
2019-08-21T19:40:51.308+0700 I SHARDING [conn5] Marking collection traveliko-analytics.hotelAnalyticsDayByCountryAndCityView as collection version: <unsharded>
2019-08-21T19:40:51.311+0700 I SHARDING [conn5] Marking collection traveliko_eventstore.snapshotevents as collection version: <unsharded>
2019-08-21T19:40:51.311+0700 I SHARDING [conn5] Marking collection traveliko_eventstore.domainevents as collection version: <unsharded>
2019-08-21T19:40:51.312+0700 I COMMAND [conn5] Finished updating version of unique indexes for upgrade, waiting for all index updates to be committed at optime { ts: Timestamp(0, 0), t: -1 }
2019-08-21T19:40:51.312+0700 F - [conn5] Invariant failure oplog src/mongo/db/transaction_history_iterator.cpp 91
2019-08-21T19:40:51.312+0700 F - [conn5]
***aborting after invariant() failure
2019-08-21T19:40:51.330+0700 F - [conn5] Got signal: 6 (Aborted).
0x555f7237db31 0x555f7237d32e 0x555f7237d3c6 0x7f06af44d890 0x7f06af088e97 0x7f06af08a801 0x555f708b6350 0x555f70726a4d 0x555f711b8e1e 0x555f711c3ff2 0x555f711afbaf 0x555f70df4846 0x555f70df6dd9 0x555f71d8d254 0x555f70cf7f09 0x555f70cf95c3 0x555f70cfa47e 0x555f70cfad50 0x555f70ce8d4c 0x555f70cf493c 0x555f70cf028f 0x555f70cf350c 0x555f71ad55e2 0x555f70cedcad 0x555f70cf0f43 0x555f70cef377 0x555f70cf01eb 0x555f70cf350c 0x555f71ad5a4b 0x555f7210fc64 0x7f06af4426db 0x7f06af16b88f
----- BEGIN BACKTRACE -----
,{"b":"555F6FBFD000","o":"278032E"},{"b":"555F6FBFD000","o":"27803C6"},{"b":"7F06AF43B000","o":"12890"},{"b":"7F06AF04A000","o":"3EE97","s":"gsignal"},{"b":"7F06AF04A000","o":"40801","s":"abort"},{"b":"555F6FBFD000","o":"CB9350","s":"ZN5mongo22invariantFailedWithMsgEPKcRKNSt7_cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j"},{"b":"555F6FBFD000","o":"B29A4D"},{"b":"555F6FBFD000","o":"15BBE1E","s":"_ZN5mongo26TransactionHistoryIterator4nextEPNS_16OperationContextE"},{"b":"555F6FBFD000","o":"15C6FF2","s":"_ZN5mongo22TransactionParticipant11Participant26refreshFromStorageIfNeededEPNS_16OperationContextE"},{"b":"555F6FBFD000","o":"15B2BAF","s":"_ZN5mongo29MongoDOperationContextSessionC2EPNS_16OperationContextE"},{"b":"555F6FBFD000","o":"11F7846"},{"b":"555F6FBFD000","o":"11F9DD9"},{"b":"555F6FBFD000","o":"2190254","s":"_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_3rpc21ReplyBuilderInterfaceE"},{"b":"555F6FBFD000","o":"10FAF09"},{"b":"555F6FBFD000","o":"10FC5C3"},{"b":"555F6FBFD000","o":"10FD47E"},{"b":"555F6FBFD000","o":"10FDD50","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE"},{"b":"555F6FBFD000","o":"10EBD4C","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"555F6FBFD000","o":"10F793C","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"555F6FBFD000","o":"10F328F","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"555F6FBFD000","o":"10F650C"},{"b":"555F6FBFD000","o":"1ED85E2","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE"},{"b":"555F6FBFD000","o":"10F0CAD","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE"},{"b":"555F6FBFD000","o":"10F3F43","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"555F6FBFD000","o":"10F2377","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"555F6FBFD000","o":"10F31EB","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"555F6FBFD000","o":"10F650C"},{"b":"555F6FBFD000","o":"1ED8A4B"},{"b":"555F6FBFD000","o":"2512C64"},{"b":"7F06AF43B000","o":"76DB"},{"b":"7F06AF04A000","o":"12188F","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.2.0", "gitVersion" : "a4b751dcf51dd249c5865812b390cfd1c0129c30", "compiledModules" : [], "uname" :
{ "sysname" : "Linux", "release" : "4.15.0-58-generic", "version" : "#64-Ubuntu SMP Tue Aug 6 11:12:41 UTC 2019", "machine" : "x86_64" }, "somap" : [ { "b" : "555F6FBFD000", "elfType" : 3, "buildId" : "D849CC14858F969B5888F892C9E16777E4C6FF32" }, { "b" : "7FFD9E2FC000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "D523EBC9181DE86DAB768A6167F8B401D3054550" }, { "b" : "7F06B098F000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "20310A72634F3373FA7BCAB64DFB64CC204554B2" }, { "b" : "7F06B0774000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "390E9CC4C215314B6D8ADE6D6E28F8518418039C" }, { "b" : "7F06B02A9000", "path" : "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1", "elfType" : 3, "buildId" : "CB6876717C83B0CC01C3C919B9B6E86D8554F546" }, { "b" : "7F06B001C000", "path" : "/usr/lib/x86_64-linux-gnu/libssl.so.1.1", "elfType" : 3, "buildId" : "439A262CC0127BA401707DEC7A28884D617550E0" }, { "b" : "7F06AFE18000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "25AD56E902E23B490A9CCDB08A9744D89CB95BCC" }, { "b" : "7F06AFC10000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "9826FBDF57ED7D6965131074CB3C08B1009C1CD8" }, { "b" : "7F06AF872000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "A33761AB8FB485311B3C85BF4253099D7CABE653" }, { "b" : "7F06AF65A000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "41BDC55C07D5E5B1D8AB38E2C19B1F535855E084" }, { "b" : "7F06AF43B000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "28C6AADE70B2D40D1F0F3D0A1A0CAD1AB816448F" }, { "b" : "7F06AF04A000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B417C0BA7CC5CF06D1D1BED6652CEDB9253C60D0" }, { "b" : "7F06B0C0E000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "64DF1B961228382FE18684249ED800AB1DCEAAD4" }, { "b" : "7F06AEE25000", "path" : "/usr/lib/x86_64-linux-gnu/libnghttp2.so.14", "elfType" : 3, "buildId" : "4F00E5207693FDC249DA42EC6472ACA6A7B929AE" }, { "b" : "7F06AEC08000", "path" : "/usr/lib/x86_64-linux-gnu/libidn2.so.0", "elfType" : 3, "buildId" : "BA5BF9A5C44F48C647E9D8270A5421AE81CCAD61" }, { "b" : "7F06AE9EC000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "69465D8AA6B19086ABF2455A703F9168BF82A69F" }, { "b" : "7F06AE7DE000", "path" : "/usr/lib/x86_64-linux-gnu/libpsl.so.5", "elfType" : 3, "buildId" : "CDAF1F1946846941F9D06414EC8C812D131A168E" }, { "b" : "7F06AE593000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "00F419F64B0E70D8C5EEF7050369AA40B2A6E090" }, { "b" : "7F06AE341000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "EC29F33C48F75AA2C48BA08E0F74D9546622314D" }, { "b" : "7F06AE133000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "25604CADD65A7057CB774139D0CBDF14460C1C78" }, { "b" : "7F06ADF16000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "EF3E006DFE3132A41D4D4DC0E407D6EA658E11C4" }, { "b" : "7F06ADB98000", "path" : "/usr/lib/x86_64-linux-gnu/libunistring.so.2", "elfType" : 3, "buildId" : "0E2784298E7D3F4D894FE130ACEFA77C3E624F72" }, { "b" : "7F06AD833000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "E5AE5C31F804BE96532D0DB2091F19E472F2D4A0" }, { "b" : "7F06AD5FF000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "842BDF0B0EAAB82E19F1EABFC38769F4040FBE31" }, { "b" : "7F06AD3C9000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "C20D4B3BA13FCDCC3BF6857689BA9FC70BE3F6A5" }, { "b" : "7F06AD148000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "D40EA9B5EC5BC46799E4A412319617BD38BE9341" }, { "b" : "7F06ACE72000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "69FBCF425EE6DF03DE93B82FBC2FC33790E68A96" }, { "b" : "7F06ACC40000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "F400D5D643A7F9696DF0E6148FA99BEE6C1BDDF7" }, { "b" : "7F06ACA3C000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "C0CB7E35A4566A443F99DFBC1A54D3A0677C8A10" }, { "b" : "7F06AC831000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "D78D71E8E016A534281B25B97CD7E5E9DB5FE00A" }, { "b" : "7F06AC616000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "ABB7E3F40302E6509DAD1F91DFB1F04B6A5FD072" }, { "b" : "7F06AC3D5000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "A1A98DB481968073636BBAECB561A3EA8ED198AE" }, { "b" : "7F06AC0A6000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "8DBD451EA5651283905E16FA7DFA9908688893A3" }, { "b" : "7F06ABE93000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "6036B89A3BB671B32E01464C0C82BFA016186352" }, { "b" : "7F06ABC8F000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "F463E107B099910463BC32E837C73D341A52C27B" }, { "b" : "7F06ABA86000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "C2376C5B831991591F1A67B976758185F86896D8" }, { "b" : "7F06AB7F9000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "69BDEE5FA0FEEDF317308BE850F78761861D520A" }, { "b" : "7F06AB557000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "315D74995AAA32DE4D15BA25F335066988B1B230" }, { "b" : "7F06AB321000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "6673972A1C24A89EBAFBAE696188A4CB26C6DDEB" }, { "b" : "7F06AB10B000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "430827C33259C12248CF44B91A9A9821114376F5" }, { "b" : "7F06AAF03000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "3555B5F599C9787DFDDBF9E8DF6F706B9044D985" }, { "b" : "7F06AACDA000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "93A0931B1C2818F0EA224CE6FE5E31E84A9B55BB" }, { "b" : "7F06AAACB000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "669D4CCE42FA4382796EFFCF0C16F459F4382C4C" }, { "b" : "7F06AA881000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "4B80C543356EE0AF9039EFE7C9EA1CC1F74C426A" }, { "b" : "7F06AA578000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "BCE351987CF42B3D258B09F0CAC867758D935086" }, { "b" : "7F06AA340000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "810686AF0D5FD350A4FB1CC4B5AFF44A05C102CB" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x555f7237db31]
mongod(+0x278032E) [0x555f7237d32e]
mongod(+0x27803C6) [0x555f7237d3c6]
libpthread.so.0(+0x12890) [0x7f06af44d890]
libc.so.6(gsignal+0xC7) [0x7f06af088e97]
libc.so.6(abort+0x141) [0x7f06af08a801]
mongod(ZN5mongo22invariantFailedWithMsgEPKcRKNSt7_cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j+0x0) [0x555f708b6350]
mongod(+0xB29A4D) [0x555f70726a4d]
mongod(_ZN5mongo26TransactionHistoryIterator4nextEPNS_16OperationContextE+0x4E) [0x555f711b8e1e]
mongod(_ZN5mongo22TransactionParticipant11Participant26refreshFromStorageIfNeededEPNS_16OperationContextE+0x802) [0x555f711c3ff2]
mongod(_ZN5mongo29MongoDOperationContextSessionC2EPNS_16OperationContextE+0x4F) [0x555f711afbaf]
mongod(+0x11F7846) [0x555f70df4846]
mongod(+0x11F9DD9) [0x555f70df6dd9]
mongod(_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_3rpc21ReplyBuilderInterfaceE+0x74) [0x555f71d8d254]
mongod(+0x10FAF09) [0x555f70cf7f09]
mongod(+0x10FC5C3) [0x555f70cf95c3]
mongod(+0x10FD47E) [0x555f70cfa47e]
mongod(_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE+0x540) [0x555f70cfad50]
mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x3C) [0x555f70ce8d4c]
mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0xEC) [0x555f70cf493c]
mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x17F) [0x555f70cf028f]
mongod(+0x10F650C) [0x555f70cf350c]
mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE+0x182) [0x555f71ad55e2]
mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE+0x10D) [0x555f70cedcad]
mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0x843) [0x555f70cf0f43]
mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x2E7) [0x555f70cef377]
mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0xDB) [0x555f70cf01eb]
mongod(+0x10F650C) [0x555f70cf350c]
mongod(+0x1ED8A4B) [0x555f71ad5a4b]
mongod(+0x2512C64) [0x555f7210fc64]
libpthread.so.0(+0x76DB) [0x7f06af4426db]
libc.so.6(clone+0x3F) [0x7f06af16b88f]
----- END BACKTRACE -----