{"t":{"$date":"2022-05-05T06:09:05.012+00:00"},"s":"I", "c":"CONTROL", "id":31380, "ctx":"conn402","msg":"BACKTRACE","attr":{"bt":{"backtrace":[{"a":"558E10F89AB4","b":"558E0BFB5000","o":"4FD4AB4","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.358","C":"mongo::stack_trace_detail::(anonymous namespace)::printStackTraceImpl(mongo::stack_trace_detail::(anonymous namespace)::Options const&, mongo::StackTraceSink*) [clone .constprop.358]","s+":"1F4"},{"a":"558E10F8BFC9","b":"558E0BFB5000","o":"4FD6FC9","s":"_ZN5mongo15printStackTraceEv","C":"mongo::printStackTrace()","s+":"29"},{"a":"558E10F85DF6","b":"558E0BFB5000","o":"4FD0DF6","s":"abruptQuit","s+":"66"},{"a":"7F8D7FA688E0","b":"7F8D7FA57000","o":"118E0","s":"funlockfile","s+":"50"},{"a":"7F8D7F6DFCA0","b":"7F8D7F6AC000","o":"33CA0","s":"gsignal","s+":"110"},{"a":"7F8D7F6E1148","b":"7F8D7F6AC000","o":"35148","s":"abort","s+":"148"},{"a":"558E0DF4B9D0","b":"558E0BFB5000","o":"1F969D0","s":"_ZN5mongo15invariantFailedEPKcS1_j","C":"mongo::invariantFailed(char const*, char const*, unsigned int)","s+":"F7"},{"a":"558E0D9846FE","b":"558E0BFB5000","o":"19CF6FE","s":"_ZN5mongo4repl26ReplicationCoordinatorImpl16awaitReplicationEPNS_16OperationContextERKNS0_6OpTimeERKNS_19WriteConcernOptionsE.cold.4083","C":"mongo::repl::ReplicationCoordinatorImpl::awaitReplication(mongo::OperationContext*, mongo::repl::OpTime const&, mongo::WriteConcernOptions const&) [clone .cold.4083]","s+":"18"},{"a":"558E0F371E07","b":"558E0BFB5000","o":"33BCE07","s":"_ZN5mongo19waitForWriteConcernEPNS_16OperationContextERKNS_4repl6OpTimeERKNS_19WriteConcernOptionsEPNS_18WriteConcernResultE","C":"mongo::waitForWriteConcern(mongo::OperationContext*, mongo::repl::OpTime const&, mongo::WriteConcernOptions const&, mongo::WriteConcernResult*)","s+":"637"},{"a":"558E0E1B16DD","b":"558E0BFB5000","o":"21FC6DD","s":"_ZZNK5mongo23ServiceEntryPointMongod5Hooks19waitForWriteConcernEPNS_16OperationContextEPKNS_17CommandInvocationERKNS_4repl6OpTimeERNS_14BSONObjBuilderEENKUlvE_clEv","C":"mongo::ServiceEntryPointMongod::Hooks::waitForWriteConcern(mongo::OperationContext*, mongo::CommandInvocation const*, mongo::repl::OpTime const&, mongo::BSONObjBuilder&) const::{lambda()#1}::operator()() const","s+":"3ED"},{"a":"558E0E1B1F73","b":"558E0BFB5000","o":"21FCF73","s":"_ZNK5mongo23ServiceEntryPointMongod5Hooks19waitForWriteConcernEPNS_16OperationContextEPKNS_17CommandInvocationERKNS_4repl6OpTimeERNS_14BSONObjBuilderE","C":"mongo::ServiceEntryPointMongod::Hooks::waitForWriteConcern(mongo::OperationContext*, mongo::CommandInvocation const*, mongo::repl::OpTime const&, mongo::BSONObjBuilder&) const","s+":"173"},{"a":"558E0E1BD71E","b":"558E0BFB5000","o":"220871E","s":"_ZN5mongo12_GLOBAL__N_132RunCommandAndWaitForWriteConcern20_waitForWriteConcernERNS_14BSONObjBuilderE","C":"mongo::(anonymous namespace)::RunCommandAndWaitForWriteConcern::_waitForWriteConcern(mongo::BSONObjBuilder&)","s+":"1AE"},{"a":"558E0E1BDB32","b":"558E0BFB5000","o":"2208B32","s":"_ZN5mongo14future_details12throwingCallIZNS_12_GLOBAL__N_132RunCommandAndWaitForWriteConcern8_runImplEvEUlNS_6StatusEE_JNS_10StatusWithINS0_8FakeVoidEEEEEEDaOT_DpOT0_","C":"auto mongo::future_details::throwingCall >(mongo::(anonymous namespace)::RunCommandAndWaitForWriteConcern::_runImpl()::{lambda(mongo::Status)#1}&&, mongo::StatusWith&&)","s+":"F2"},{"a":"558E0E1BDD98","b":"558E0BFB5000","o":"2208D98","s":"_ZZNO5mongo14future_details10FutureImplINS0_8FakeVoidEE12onCompletionIZNS_12_GLOBAL__N_132RunCommandAndWaitForWriteConcern8_runImplEvEUlNS_6StatusEE_EEDaOT_ENKUlOS2_E_clESB_.isra.2014","C":"mongo::future_details::FutureImpl::onCompletion(mongo::(anonymous namespace)::RunCommandAndWaitForWriteConcern::_runImpl()::{lambda(mongo::Status)#1}&&) &&::{lambda(mongo::future_details::FakeVoid&&)#1}::operator()(mongo::(anonymous namespace)::RunCommandAndWaitForWriteConcern::_runImpl()::{lambda(mongo::Status)#1}&&) const [clone .isra.2014]","s+":"38"},{"a":"558E0E1C73E0","b":"558E0BFB5000","o":"22123E0","s":"_ZN5mongo12_GLOBAL__N_132RunCommandAndWaitForWriteConcern8_runImplEv","C":"mongo::(anonymous namespace)::RunCommandAndWaitForWriteConcern::_runImpl()","s+":"120"},{"a":"558E0E1B7ADC","b":"558E0BFB5000","o":"2202ADC","s":"_ZN5mongo12_GLOBAL__N_114RunCommandImpl3runEv","C":"mongo::(anonymous namespace)::RunCommandImpl::run()","s+":"13C"},{"a":"558E0E1BE5E4","b":"558E0BFB5000","o":"22095E4","s":"_ZN5mongo12_GLOBAL__N_119ExecCommandDatabase12_commandExecEv","C":"mongo::(anonymous namespace)::ExecCommandDatabase::_commandExec()","s+":"1E4"},{"a":"558E0E1C302F","b":"558E0BFB5000","o":"220E02F","s":"_ZZN5mongo12_GLOBAL__N_114executeCommandESt10shared_ptrINS0_13HandleRequest16ExecutionContextEEENUlvE0_clEv","C":"mongo::(anonymous namespace)::executeCommand(std::shared_ptr)::{lambda()#2}::operator()()","s+":"43F"},{"a":"558E0E1C4FA0","b":"558E0BFB5000","o":"220FFA0","s":"_ZZN5mongo15unique_functionIFvPNS_14future_details15SharedStateBaseEEE8makeImplIZNS1_10FutureImplINS1_8FakeVoidEE16makeContinuationIvZZNOS9_4thenIZNS_12_GLOBAL__N_114executeCommandESt10shared_ptrINSC_13HandleRequest16ExecutionContextEEEUlvE0_EEDaOT_ENKUlvE1_clEvEUlPNS1_15SharedStateImplIS8_EESN_E_EENS7_ISI_EEOT0_EUlS3_E_EEDaSJ_EN12SpecificImpl4callEOS3_","C":"mongo::unique_function::makeImpl::makeContinuation::then)::{lambda()#2}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr)::{lambda()#2}&&) &&::{lambda()#3}::operator()() const::{lambda(mongo::future_details::SharedStateImpl*, mongo::future_details::SharedStateImpl)#1}>(mongo::future_details::FutureImpl::then)::{lambda()#2}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr)::{lambda()#2}&&) &&::{lambda()#3}::operator()() const::{lambda(mongo::future_details::SharedStateImpl*, mongo::future_details::SharedStateImpl)#1}&&)::{lambda(mongo::future_details::SharedStateBase*)#1}>(mongo::future_details::FutureImpl::makeContinuation::then)::{lambda()#2}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr)::{lambda()#2}&&) &&::{lambda()#3}::operator()() const::{lambda(mongo::future_details::SharedStateImpl*, mongo::future_details::SharedStateImpl)#1}>(mongo::future_details::FutureImpl::then)::{lambda()#2}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr)::{lambda()#2}&&) &&::{lambda()#3}::operator()() const::{lambda(mongo::future_details::SharedStateImpl*, mongo::future_details::SharedStateImpl)#1}&&)::{lambda(mongo::future_details::SharedStateBase*)#1})::SpecificImpl::call(mongo::future_details::SharedStateBase*&&)","s+":"80"},{"a":"558E0E186FD7","b":"558E0BFB5000","o":"21D1FD7","s":"_ZN5mongo14future_details15SharedStateBase20transitionToFinishedEv","C":"mongo::future_details::SharedStateBase::transitionToFinished()","s+":"147"},{"a":"558E0E1C9012","b":"558E0BFB5000","o":"2214012","s":"_ZN5mongo14future_details10FutureImplINS0_8FakeVoidEE11generalImplIZNOS3_17propagateResultToEPNS0_15SharedStateImplIS2_EEEUlOS2_E_ZNOS3_17propagateResultToES7_EUlONS_6StatusEE0_ZNOS3_17propagateResultToES7_EUlvE1_EEDaOT_OT0_OT1_","C":"auto mongo::future_details::FutureImpl::generalImpl::propagateResultTo(mongo::future_details::SharedStateImpl*) &&::{lambda(mongo::future_details::FakeVoid&&)#1}, mongo::future_details::FutureImpl::propagateResultTo(mongo::future_details::SharedStateImpl*) &&::{lambda(mongo::Status&&)#2}, mongo::future_details::FutureImpl::propagateResultTo(mongo::future_details::SharedStateImpl*) &&::{lambda()#3}>(mongo::future_details::FutureImpl::propagateResultTo(mongo::future_details::SharedStateImpl*) &&::{lambda(mongo::future_details::FakeVoid&&)#1}&&, mongo::future_details::FutureImpl::propagateResultTo(mongo::future_details::SharedStateImp... Collapse 1:28 Part of that backtrace is setFeatureCompatibilityVersion 1:28 SetFeatureCompatibilityVersionCommand3runEPNS_16OperationContextER peter.vertenten 1:37 PM And the other important part, the agent does confirm it as rc4 1:37 6.0.0-rc4-ff5e921b9378126dc1a44ae4d783d39afde09a7a(enterprise) 1:37 As well as the mongod 1:37 {"t":{"$date":"2022-05-05T06:08:10.625+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"6.0.0-rc4","gitVersion":"ff5e921b9378126dc1a44ae4d783d39afde09a7a","openSSLVersion":"OpenSSL 1.0.2k-fips 26 Jan 2017","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"amazon2","distarch":"x86_64","target_arch":"x86_64"}}}} 1:38 will double check that it is the same invariant as before peter.vertenten 1:43 PM So based on the logs it looks like the same invariant. 1:44 {"t":{"$date":"2022-05-05T06:09:04.746+00:00"},"s":"F", "c":"ASSERT", "id":23079, "ctx":"conn402","msg":"Invariant failure","attr":{"expr":"!opCtx->lockState()->isLocked()","file":"src/mongo/db/repl/replication_coordinator_impl.cpp","line":2017}}