Hi Team,

 

We initiated Y2038 testing by advancing the system clock to 2038-01-19T01:00:00. Initially, the replicas were functioning healthily, servicing traffic without issues. However, as the system approached the Y2038 time threshold, MongoDB experienced a sudden crash, accompanied by the following error messages in the primary node

Test Procedure is simple

1) Have a PSA

2) set the clock to 2038-01-19T01:00:00 and observe the replicaSet health

3) once the system clock moves beyond  2038-01-19T03:14:07 observe the crash

We are using v5.0.20

{"t":\{"$date":"2038-01-19T03:14:10.057+00:00"}

,"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn13180","msg":"client metadata","attr":{"remote":"172.20.35.88:54892","client":"conn13180","doc":{"application":

{"name":"MongoDB Shell"}

,"driver":{"name":"MongoDB Internal Client","version":"5.0.20"},"os":{"type":"Linux","name":"AlmaLinux release 8.8 (Sapphire Caracal)","architecture":"x86_64","version":"Kernel 4.18.0-477.27.1.el8_8.x86_64"}}}}

{"t":\{"$date":"2038-01-19T03:14:10.090+00:00"}

,"s":"I",  "c":"-",        "id":20883,   "ctx":"conn13178","msg":"Interrupted operation as its client disconnected","attr":{"opId":230177}}

{"t":\{"$date":"2038-01-19T03:14:10.109+00:00"}

,"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn13181","msg":"client metadata","attr":{"remote":"172.20.35.88:54906","client":"conn13181","doc":{"application":

{"name":"MongoDB Shell"}

,"driver":{"name":"MongoDB Internal Client","version":"5.0.20"},"os":{"type":"Linux","name":"AlmaLinux release 8.8 (Sapphire Caracal)","architecture":"x86_64","version":"Kernel 4.18.0-477.27.1.el8_8.x86_64"}}}}

{"t":\{"$date":"2038-01-19T03:14:10.233+00:00"}

,"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"NoopWriter","msg":"Writing fatal message","attr":{"message":"terminate() called. An exception is active; attempting to gather more information\n"}}

{"t":\{"$date":"2038-01-19T03:14:10.233+00:00"}

,"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"NoopWriter","msg":"Writing fatal message","attr":{"message":"DBException::toString(): Location40482: $clusterTime cannot be advanced beyond the maximum logical time value\nActual exception type: mongo::error_details::throwExceptionForStatus(mongo::Status const&)::NonspecificAssertionException\n\n"}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31380,   "ctx":"NoopWriter","msg":"BACKTRACE","attr":{"bt":{"backtrace":[

{"a":"55B48376E365","b":"55B47F7F2000","o":"3F7C365","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.361","s+":"215"}

,{"a":"55B483770DE9","b":"55B47F7F2000","o":"3F7EDE9","s":"ZN5mongo15printStackTraceEv","s+":"29"},{"a":"55B483769907","b":"55B47F7F2000","o":"3F77907","s":"_ZN5mongo12_GLOBAL_N_111myTerminateEv","s+":"D7"},{"a":"55B483903BB6","b":"55B47F7F2000","o":"4111BB6","s":"ZN10cxxabiv111terminateEPFvvE","s+":"6"},{"a":"55B483903BF1","b":"55B47F7F2000","o":"4111BF1","s":"_ZSt9terminatev","s+":"11"},{"a":"55B483903D24","b":"55B47F7F2000","o":"4111D24","s":"cxa_throw","s+":"44"},{"a":"55B480BF6B2F","b":"55B47F7F2000","o":"1404B2F","s":"_ZN5mongo13error_details23throwExceptionForStatusERKNS_6StatusE","s+":"8DCF"},{"a":"55B480C068B3","b":"55B47F7F2000","o":"14148B3","s":"_ZN5mongo21uassertedWithLocationERKNS_6StatusEPKcj","s+":"337"},{"a":"55B480A3A571","b":"55B47F7F2000","o":"1248571","s":"_ZN5mongo18VectorClockMutable28_advanceComponentTimeByTicksENS_11VectorClock9ComponentEm.cold.404","s+":"93"},{"a":"55B4824352FE","b":"55B47F7F2000","o":"2C432FE","s":"_ZN5mongo14LocalOplogInfo14getNextOpTimesEPNS_16OperationContextEm","s+":"AE"},{"a":"55B481C56D88","b":"55B47F7F2000","o":"2464D88","s":"_ZN5mongo4repl5logOpEPNS_16OperationContextEPNS0_17MutableOplogEntryE","s+":"158"},{"a":"55B4816777DB","b":"55B47F7F2000","o":"1E857DB","s":"_ZN5mongo12_GLOBAL_N_112logOperationEPNS_16OperationContextEPNS_4repl17MutableOplogEntryE","s+":"4B"},{"a":"55B48167B861","b":"55B47F7F2000","o":"1E89861","s":"ZN5mongo14OpObserverImpl19onInternalOpMessageEPNS_16OperationContextERKNS_15NamespaceStringEN5boost8optionalINS_4UUIDEEERKNS_7BSONObjENS7_ISA_EENS7_INS_4repl6OpTimeEEESG_SG_SG","s+":"231"},{"a":"55B480D90CCA","b":"55B47F7F2000","o":"159ECCA","s":"ZN5mongo18OpObserverRegistry19onInternalOpMessageEPNS_16OperationContextERKNS_15NamespaceStringEN5boost8optionalINS_4UUIDEEERKNS_7BSONObjENS7_ISA_EENS7_INS_4repl6OpTimeEEESG_SG_SG","s+":"21A"},{"a":"55B4810B18C7","b":"55B47F7F2000","o":"18BF8C7","s":"_ZZN5mongo4repl10NoopWriter10_writeNoopEPNS_16OperationContextEENKUlvE3_clEv.isra.299","s+":"D7"},{"a":"55B4810B2069","b":"55B47F7F2000","o":"18C0069","s":"_ZN5mongo4repl10NoopWriter10_writeNoopEPNS_16OperationContextE","s+":"719"},{"a":"55B4810B367B","b":"55B47F7F2000","o":"18C167B","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_4repl10NoopWriter18PeriodicNoopRunnerC4ENS3_8DurationISt5ratioILl1ELl1EEEESt8functionIFvPNS3_16OperationContextEEEEUlvE_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"3BB"},{"a":"55B48391FE4F","b":"55B47F7F2000","o":"412DE4F","s":"execute_native_thread_routine","s+":"F"},{"a":"7FCBBDD811CA","b":"7FCBBDD79000","o":"81CA","s":"start_thread","s+":"EA"},{"a":"7FCBBD9EDE73","b":"7FCBBD9B4000","o":"39E73","s":"clone","s+":"43"}],"processInfo":{"mongodbVersion":"5.0.20","gitVersion":"2cd626d8148120319d7dca5824e760fe220cb0de","compiledModules":[],"uname":

{"sysname":"Linux","release":"4.18.0-477.27.1.el8_8.x86_64","version":"#1 SMP Thu Sep 21 06:49:25 EDT 2023","machine":"x86_64"}

,"somap":[\{"b":"55B47F7F2000","elfType":3,"buildId":"A8EA7166EFC23E0D3802F8AFEFEF2186CF5E5BBD"},\{"b":"7FCBBDD79000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"76F163FDBAA9E91050B456A7E5EA8AC78563BD29"},\{"b":"7FCBBD9B4000","path":"/lib64/libc.so.6","elfType":3,"buildId":"44ED73CF68E8FA608DA3B301146C81A0A77A5619"}]}}}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B48376E365","b":"55B47F7F2000","o":"3F7C365","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.361","s+":"215"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B483770DE9","b":"55B47F7F2000","o":"3F7EDE9","s":"_ZN5mongo15printStackTraceEv","s+":"29"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B483769907","b":"55B47F7F2000","o":"3F77907","s":"_ZN5mongo12_GLOBAL__N_111myTerminateEv","s+":"D7"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B483903BB6","b":"55B47F7F2000","o":"4111BB6","s":"_ZN10__cxxabiv111__terminateEPFvvE","s+":"6"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B483903BF1","b":"55B47F7F2000","o":"4111BF1","s":"_ZSt9terminatev","s+":"11"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B483903D24","b":"55B47F7F2000","o":"4111D24","s":"__cxa_throw","s+":"44"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B480BF6B2F","b":"55B47F7F2000","o":"1404B2F","s":"_ZN5mongo13error_details23throwExceptionForStatusERKNS_6StatusE","s+":"8DCF"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B480C068B3","b":"55B47F7F2000","o":"14148B3","s":"_ZN5mongo21uassertedWithLocationERKNS_6StatusEPKcj","s+":"337"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B480A3A571","b":"55B47F7F2000","o":"1248571","s":"_ZN5mongo18VectorClockMutable28_advanceComponentTimeByTicksENS_11VectorClock9ComponentEm.cold.404","s+":"93"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B4824352FE","b":"55B47F7F2000","o":"2C432FE","s":"_ZN5mongo14LocalOplogInfo14getNextOpTimesEPNS_16OperationContextEm","s+":"AE"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B481C56D88","b":"55B47F7F2000","o":"2464D88","s":"_ZN5mongo4repl5logOpEPNS_16OperationContextEPNS0_17MutableOplogEntryE","s+":"158"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B4816777DB","b":"55B47F7F2000","o":"1E857DB","s":"_ZN5mongo12_GLOBAL__N_112logOperationEPNS_16OperationContextEPNS_4repl17MutableOplogEntryE","s+":"4B"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B48167B861","b":"55B47F7F2000","o":"1E89861","s":"_ZN5mongo14OpObserverImpl19onInternalOpMessageEPNS_16OperationContextERKNS_15NamespaceStringEN5boost8optionalINS_4UUIDEEERKNS_7BSONObjENS7_ISA_EENS7_INS_4repl6OpTimeEEESG_SG_SG_","s+":"231"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B480D90CCA","b":"55B47F7F2000","o":"159ECCA","s":"_ZN5mongo18OpObserverRegistry19onInternalOpMessageEPNS_16OperationContextERKNS_15NamespaceStringEN5boost8optionalINS_4UUIDEEERKNS_7BSONObjENS7_ISA_EENS7_INS_4repl6OpTimeEEESG_SG_SG_","s+":"21A"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B4810B18C7","b":"55B47F7F2000","o":"18BF8C7","s":"_ZZN5mongo4repl10NoopWriter10_writeNoopEPNS_16OperationContextEENKUlvE3_clEv.isra.299","s+":"D7"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B4810B2069","b":"55B47F7F2000","o":"18C0069","s":"_ZN5mongo4repl10NoopWriter10_writeNoopEPNS_16OperationContextE","s+":"719"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B4810B367B","b":"55B47F7F2000","o":"18C167B","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_4repl10NoopWriter18PeriodicNoopRunnerC4ENS3_8DurationISt5ratioILl1ELl1EEEESt8functionIFvPNS3_16OperationContextEEEEUlvE_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"3BB"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"55B48391FE4F","b":"55B47F7F2000","o":"412DE4F","s":"execute_native_thread_routine","s+":"F"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"7FCBBDD811CA","b":"7FCBBDD79000","o":"81CA","s":"start_thread","s+":"EA"}

}}

{"t":\{"$date":"2038-01-19T03:14:10.360+00:00"}

,"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"NoopWriter","msg":"Frame","attr":{"frame":

{"a":"7FCBBD9EDE73","b":"7FCBBD9B4000","o":"39E73","s":"clone","s+":"43"}

}}

{"t":\{"$date":"2038-01-19T03:14:18.915+00:00"}

,"s":"I",  "c":"CONTROL",  "id":20698,   "ctx":"-","msg":"**** SERVER RESTARTED ****"}

 

db.serverBuildInfo()
{
“version” : “5.0.20”,
“gitVersion” : “2cd626d8148120319d7dca5824e760fe220cb0de”,
“modules” : ,
“allocator” : “tcmalloc”,
“javascriptEngine” : “mozjs”,
“sysInfo” : “deprecated”,
“versionArray” : [
5,
0,
20,
0
],
“openssl” :

{ “running” : “OpenSSL 1.1.1k FIPS 25 Mar 2021”, “compiled” : “OpenSSL 1.1.1 FIPS 11 Sep 2018” }

,
“buildEnvironment” : {
“distmod” : “rhel80”,
“distarch” : “x86_64”,
“cc” : “/opt/mongodbtoolchain/v3/bin/gcc: gcc (GCC) 8.5.0”,
“ccflags” : “-Werror -include mongo/platform/basic.h -ffp-contract=off -fasynchronous-unwind-tables -ggdb -Wall -Wsign-compare -Wno-unknown-pragmas -Winvalid-pch -fno-omit-frame-pointer -fno-strict-aliasing -O2 -march=sandybridge -mtune=generic -mprefer-vector-width=128 -Wno-unused-local-typedefs -Wno-unused-function -Wno-deprecated-declarations -Wno-unused-const-variable -Wno-unused-but-set-variable -Wno-missing-braces -fstack-protector-strong -Wa,–nocompress-debug-sections -fno-builtin-memcmp”,
“cxx” : “/opt/mongodbtoolchain/v3/bin/g+: g+ (GCC) 8.5.0”,
“cxxflags” : “-Woverloaded-virtual -Wno-maybe-uninitialized -fsized-deallocation -std=c++17”,
“linkflags” : “-Wl,–fatal-warnings -pthread -Wl,-z,now -fuse-ld=gold -fstack-protector-strong -Wl,–no-threads -Wl,–build-id -Wl,–hash-style=gnu -Wl,-z,noexecstack -Wl,–warn-execstack -Wl,-z,relro -Wl,–compress-debug-sections=none -Wl,-z,origin -Wl,–enable-new-dtags”,
“target_arch” : “x86_64”,
“target_os” : “linux”,
“cppdefines” : “SAFEINT_USE_INTRINSICS 0 PCRE_STATIC NDEBUG _XOPEN_SOURCE 700 _GNU_SOURCE _FORTIFY_SOURCE 2 BOOST_THREAD_VERSION 5 BOOST_THREAD_USES_DATETIME BOOST_SYSTEM_NO_DEPRECATED BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS BOOST_ENABLE_ASSERT_DEBUG_HANDLER BOOST_LOG_NO_SHORTHAND_NAMES BOOST_LOG_USE_NATIVE_SYSLOG BOOST_LOG_WITHOUT_THREAD_ATTR ABSL_FORCE_ALIGNED_ACCESS”
},
“bits” : 64,
“debug” : false,
“maxBsonObjectSize” : 16777216,
“storageEngines” : [
“devnull”,
“ephemeralForTest”,
“wiredTiger”
],
“ok” : 1
}