Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-53357

Windows latest (v4.9) replica set crashes in pymongo test suite

XMLWordPrintableJSON

    • Type: Icon: Bug Bug
    • Resolution: Fixed
    • Priority: Icon: Blocker - P1 Blocker - P1
    • 4.9.0
    • Affects Version/s: None
    • Component/s: None
    • None
    • Fully Compatible
    • Windows

      PyMongo's test suite consistently causes mongod to crash on Windows with v4.9 latest. For example: https://evergreen.mongodb.com/task/mongo_python_driver_tests_windows_python_version__platform~windows_64_vsMulti_small_auth_ssl~noauth_nossl_python_version_windows~3.9_test_latest_replica_set_61232b7f21d8f0ad9023c8ef0912017d062fafc0_20_12_08_19_37_36

      DB version:

       [2020/12/14 19:48:30.027] + ./mongodb/bin/mongod --version
 [2020/12/14 19:48:30.160] db version v4.9.0-alpha-1003-g0c4247a
 [2020/12/14 19:48:30.160] Build Info: {
 [2020/12/14 19:48:30.160]     "version": "4.9.0-alpha-1003-g0c4247a",
 [2020/12/14 19:48:30.160]     "gitVersion": "0c4247a3401c21b59f7f6553b169aecbf824091f",
 [2020/12/14 19:48:30.160]     "modules": [
 [2020/12/14 19:48:30.160]         "enterprise"
 [2020/12/14 19:48:30.160]     ],
 [2020/12/14 19:48:30.160]     "allocator": "tcmalloc",
 [2020/12/14 19:48:30.160]     "environment": {
 [2020/12/14 19:48:30.160]         "distmod": "windows",
 [2020/12/14 19:48:30.160]         "distarch": "x86_64",
 [2020/12/14 19:48:30.160]         "target_arch": "x86_64"
 [2020/12/14 19:48:30.160]     }
 [2020/12/14 19:48:30.160] }

      Crash:

      {"t":{"$date":"2020-12-14T19:51:45.241+00:00"},"s":"F",  "c":"CONTROL",  "id":23134,   "ctx":"MirrorMaestro-0","msg":"Unhandled exception","attr":{"exceptionString":"0xE0000001","addressString":"0x00007FFA546596C9"}}
{"t":{"$date":"2020-12-14T19:51:45.241+00:00"},"s":"F",  "c":"CONTROL",  "id":23136,   "ctx":"MirrorMaestro-0","msg":"*** stack trace for unhandled exception:"}
{"t":{"$date":"2020-12-14T19:51:45.245+00:00"},"s":"I",  "c":"CONTROL",  "id":31380,   "ctx":"MirrorMaestro-0","msg":"BACKTRACE","attr":{"bt":{"backtrace":[{"a":"7FFA546596C9","module":"KERNELBASE.dll","s":"RaiseException","s+":"69"},{"a":"7FF6F459ACE9","module":"mongod.exe","file":".../src/mongo/util/signal_handlers_synchronous.cpp","line":97,"s":"mongo::`anonymous namespace'::endProcessWithSignal","s+":"19"},{"a":"7FF6F459B57D","module":"mongod.exe","file":".../src/mongo/util/signal_handlers_synchronous.cpp","line":257,"s":"mongo::`anonymous namespace'::myTerminate","s+":"13D"},{"a":"7FF6F468D417","module":"mongod.exe","file":".../src/mongo/stdx/set_terminate_internals.cpp","line":88,"s":"mongo::stdx::dispatch_impl","s+":"17"},{"a":"7FF6F468D3F9","module":"mongod.exe","file":".../src/mongo/stdx/set_terminate_internals.cpp","line":92,"s":"mongo::stdx::TerminateHandlerDetailsInterface::dispatch","s+":"9"},{"a":"7FFA5456DE58","module":"ucrtbase.dll","s":"terminate","s+":"18"},{"a":"7FFA4E6A1ABF","module":"VCRUNTIME140_1.dll","s":"_NLG_Return2","s+":"96F"},{"a":"7FFA4E6A232B","module":"VCRUNTIME140_1.dll","s":"_NLG_Return2","s+":"11DB"},{"a":"7FFA4E6A40E9","module":"VCRUNTIME140_1.dll","s":"_CxxFrameHandler4","s+":"A9"},{"a":"7FF6F46C4814","module":"mongod.exe","file":"d:/A01/_work/6/s/src/vctools/crt/vcstartup/src/gs/amd64/gshandlereh4.cpp","line":86,"s":"__GSHandlerCheck_EH4","s+":"64"},{"a":"7FFA582041BF","module":"ntdll.dll","s":"_chkstk","s+":"11F"},{"a":"7FFA58164BEF","module":"ntdll.dll","s":"RtlWalkFrameChain","s+":"14BF"},{"a":"7FFA581689E6","module":"ntdll.dll","s":"RtlRaiseException","s+":"316"},{"a":"7FFA546596C9","module":"KERNELBASE.dll","s":"RaiseException","s+":"69"},{"a":"7FFA4D616220","module":"VCRUNTIME140.dll","s":"CxxThrowException","s+":"90"},{"a":"7FF6F46054AB","module":"mongod.exe","file":"C:/data/mci/34699aa8e72a3ba833cbfe3b72c5af18/src/build/cached/mongo/base/error_codes.cpp","line":2766,"s":"mongo::error_details::throwExceptionForStatus","s+":"25AB"},{"a":"7FF6F45A622B","module":"mongod.exe","file":".../src/mongo/util/assert_util.cpp","line":283,"s":"mongo::tassertFailedWithLocation","s+":"FB"},{"a":"7FF6F45A626A","module":"mongod.exe","file":".../src/mongo/util/assert_util.cpp","line":289,"s":"mongo::tassertWithLocation","s+":"3A"},{"a":"7FF6F46EC480","module":"mongod.exe","file":".../src/mongo/db/mirror_maestro.cpp","line":416,"s":"`mongo::`anonymous namespace'::MirrorMaestroImpl::_mirror'::`1'::catch$66","s+":"60"},{"a":"7FFA4E6A1080","module":"VCRUNTIME140_1.dll"},{"a":"7FFA4E6A26B5","module":"VCRUNTIME140_1.dll","s":"_NLG_Return2","s+":"1565"},{"a":"7FFA58203706","module":"ntdll.dll","s":"RtlCaptureContext","s+":"566"},{"a":"7FF6F2F7D5C8","module":"mongod.exe","file":".../src/mongo/db/mirror_maestro.cpp","line":348,"s":"mongo::`anonymous namespace'::MirrorMaestroImpl::_mirror","s+":"78"},{"a":"7FF6F2F7CF49","module":"mongod.exe","file":".../src/mongo/util/future.h","line":636,"s":"<lambda_d01db1bfc43f2924204d828948c1a2ec>::operator()","s+":"89"},{"a":"7FF6F2F7E1E1","module":"mongod.exe","file":".../src/mongo/util/functional.h","line":154,"s":"`mongo::unique_function<void __cdecl(mongo::Status)>::makeImpl<<lambda_d01db1bfc43f2924204d828948c1a2ec> >'::`2'::SpecificImpl::call","s+":"21"},{"a":"7FF6F42FFF5A","module":"mongod.exe","file":".../src/mongo/util/functional.h","line":154,"s":"`mongo::unique_function<void __cdecl(mongo::executor::TaskExecutor::CallbackArgs const &)>::makeImpl<<lambda_9d899684b466fa9c0d3fdd52f322ec66> >'::`2'::SpecificImpl::call","s+":"3A"},{"a":"7FF6F421CD27","module":"mongod.exe","file":".../src/mongo/executor/thread_pool_task_executor.cpp","line":650,"s":"mongo::executor::ThreadPoolTaskExecutor::runCallback","s+":"1E7"},{"a":"7FF6F421BA72","module":"mongod.exe","file":".../src/mongo/util/functional.h","line":154,"s":"`mongo::unique_function<void __cdecl(mongo::Status)>::makeImpl<<lambda_5b50b383851a71d8f8cc13cae1437bfc> >'::`2'::SpecificImpl::call","s+":"92"},{"a":"7FF6F44B23AF","module":"mongod.exe","file":".../src/mongo/util/concurrency/thread_pool.cpp","line":522,"s":"mongo::ThreadPool::Impl::_doOneTask","s+":"24F"},{"a":"7FF6F44B1707","module":"mongod.exe","file":".../src/mongo/util/concurrency/thread_pool.cpp","line":411,"s":"mongo::ThreadPool::Impl::_consumeTasks","s+":"97"},{"a":"7FF6F44B34C7","module":"mongod.exe","file":".../src/mongo/util/concurrency/thread_pool.cpp","line":398,"s":"mongo::ThreadPool::Impl::_workerThreadBody","s+":"1B7"},{"a":"7FF6F44AFA34","module":"mongod.exe","file":"C:/Program Files (x86)/Microsoft Visual Studio/2019/Professional/VC/Tools/MSVC/14.26.28801/include/thread","line":44,"s":"std::thread::_Invoke<std::tuple<<lambda_2a0d41949c38dda60a47c7b06190042e> >,0>","s+":"34"},{"a":"7FFA5452268A","module":"ucrtbase.dll","s":"o_exp","s+":"5A"},{"a":"7FFA554A7974","module":"KERNEL32.DLL","s":"BaseThreadInitThunk","s+":"14"}]}}}
{"t":{"$date":"2020-12-14T19:51:45.245+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA546596C9","module":"KERNELBASE.dll","s":"RaiseException","s+":"69"}}}
{"t":{"$date":"2020-12-14T19:51:45.245+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F459ACE9","module":"mongod.exe","file":".../src/mongo/util/signal_handlers_synchronous.cpp","line":97,"s":"mongo::`anonymous namespace'::endProcessWithSignal","s+":"19"}}}
{"t":{"$date":"2020-12-14T19:51:45.245+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F459B57D","module":"mongod.exe","file":".../src/mongo/util/signal_handlers_synchronous.cpp","line":257,"s":"mongo::`anonymous namespace'::myTerminate","s+":"13D"}}}
{"t":{"$date":"2020-12-14T19:51:45.245+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F468D417","module":"mongod.exe","file":".../src/mongo/stdx/set_terminate_internals.cpp","line":88,"s":"mongo::stdx::dispatch_impl","s+":"17"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F468D3F9","module":"mongod.exe","file":".../src/mongo/stdx/set_terminate_internals.cpp","line":92,"s":"mongo::stdx::TerminateHandlerDetailsInterface::dispatch","s+":"9"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA5456DE58","module":"ucrtbase.dll","s":"terminate","s+":"18"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA4E6A1ABF","module":"VCRUNTIME140_1.dll","s":"_NLG_Return2","s+":"96F"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA4E6A232B","module":"VCRUNTIME140_1.dll","s":"_NLG_Return2","s+":"11DB"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA4E6A40E9","module":"VCRUNTIME140_1.dll","s":"_CxxFrameHandler4","s+":"A9"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F46C4814","module":"mongod.exe","file":"d:/A01/_work/6/s/src/vctools/crt/vcstartup/src/gs/amd64/gshandlereh4.cpp","line":86,"s":"__GSHandlerCheck_EH4","s+":"64"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA582041BF","module":"ntdll.dll","s":"_chkstk","s+":"11F"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA58164BEF","module":"ntdll.dll","s":"RtlWalkFrameChain","s+":"14BF"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA581689E6","module":"ntdll.dll","s":"RtlRaiseException","s+":"316"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA546596C9","module":"KERNELBASE.dll","s":"RaiseException","s+":"69"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA4D616220","module":"VCRUNTIME140.dll","s":"CxxThrowException","s+":"90"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F46054AB","module":"mongod.exe","file":"C:/data/mci/34699aa8e72a3ba833cbfe3b72c5af18/src/build/cached/mongo/base/error_codes.cpp","line":2766,"s":"mongo::error_details::throwExceptionForStatus","s+":"25AB"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F45A622B","module":"mongod.exe","file":".../src/mongo/util/assert_util.cpp","line":283,"s":"mongo::tassertFailedWithLocation","s+":"FB"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F45A626A","module":"mongod.exe","file":".../src/mongo/util/assert_util.cpp","line":289,"s":"mongo::tassertWithLocation","s+":"3A"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F46EC480","module":"mongod.exe","file":".../src/mongo/db/mirror_maestro.cpp","line":416,"s":"`mongo::`anonymous namespace'::MirrorMaestroImpl::_mirror'::`1'::catch$66","s+":"60"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA4E6A1080","module":"VCRUNTIME140_1.dll"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA4E6A26B5","module":"VCRUNTIME140_1.dll","s":"_NLG_Return2","s+":"1565"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA58203706","module":"ntdll.dll","s":"RtlCaptureContext","s+":"566"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F2F7D5C8","module":"mongod.exe","file":".../src/mongo/db/mirror_maestro.cpp","line":348,"s":"mongo::`anonymous namespace'::MirrorMaestroImpl::_mirror","s+":"78"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F2F7CF49","module":"mongod.exe","file":".../src/mongo/util/future.h","line":636,"s":"<lambda_d01db1bfc43f2924204d828948c1a2ec>::operator()","s+":"89"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F2F7E1E1","module":"mongod.exe","file":".../src/mongo/util/functional.h","line":154,"s":"`mongo::unique_function<void __cdecl(mongo::Status)>::makeImpl<<lambda_d01db1bfc43f2924204d828948c1a2ec> >'::`2'::SpecificImpl::call","s+":"21"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F42FFF5A","module":"mongod.exe","file":".../src/mongo/util/functional.h","line":154,"s":"`mongo::unique_function<void __cdecl(mongo::executor::TaskExecutor::CallbackArgs const &)>::makeImpl<<lambda_9d899684b466fa9c0d3fdd52f322ec66> >'::`2'::SpecificImpl::call","s+":"3A"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F421CD27","module":"mongod.exe","file":".../src/mongo/executor/thread_pool_task_executor.cpp","line":650,"s":"mongo::executor::ThreadPoolTaskExecutor::runCallback","s+":"1E7"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F421BA72","module":"mongod.exe","file":".../src/mongo/util/functional.h","line":154,"s":"`mongo::unique_function<void __cdecl(mongo::Status)>::makeImpl<<lambda_5b50b383851a71d8f8cc13cae1437bfc> >'::`2'::SpecificImpl::call","s+":"92"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F44B23AF","module":"mongod.exe","file":".../src/mongo/util/concurrency/thread_pool.cpp","line":522,"s":"mongo::ThreadPool::Impl::_doOneTask","s+":"24F"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F44B1707","module":"mongod.exe","file":".../src/mongo/util/concurrency/thread_pool.cpp","line":411,"s":"mongo::ThreadPool::Impl::_consumeTasks","s+":"97"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F44B34C7","module":"mongod.exe","file":".../src/mongo/util/concurrency/thread_pool.cpp","line":398,"s":"mongo::ThreadPool::Impl::_workerThreadBody","s+":"1B7"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FF6F44AFA34","module":"mongod.exe","file":"C:/Program Files (x86)/Microsoft Visual Studio/2019/Professional/VC/Tools/MSVC/14.26.28801/include/thread","line":44,"s":"std::thread::_Invoke<std::tuple<<lambda_2a0d41949c38dda60a47c7b06190042e> >,0>","s+":"34"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA5452268A","module":"ucrtbase.dll","s":"o_exp","s+":"5A"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"MirrorMaestro-0","msg":"Frame","attr":{"frame":{"a":"7FFA554A7974","module":"KERNEL32.DLL","s":"BaseThreadInitThunk","s+":"14"}}}
{"t":{"$date":"2020-12-14T19:51:45.246+00:00"},"s":"I",  "c":"CONTROL",  "id":23132,   "ctx":"MirrorMaestro-0","msg":"Writing minidump diagnostic file","attr":{"dumpName":"C:\\data\\mci\\a2645657451d5118ee0154b11756ad0c\\drivers-tools\\mongodb\\bin\\mongod.2020-12-14T19-51-45.mdmp"}}
{"t":{"$date":"2020-12-14T19:51:45.248+00:00"},"s":"D1", "c":"REPL",     "id":21337,   "ctx":"conn260","msg":"Required snapshot optime is not yet part of the current 'committed' snapshot","attr":{"opTime":{"ts":{"$timestamp":{"t":1607975505,"i":23}},"t":2},"currentCommittedSnapshotOpTime":{"ts":{"$timestamp":{"t":1607975505,"i":22}},"t":2}}}

      Full mini dump and mongod logs for the above patch build are attached:

        1. db-logs.zip
          1.93 MB
        2. mongod.2020-12-14T19-51-45.mdmp.zip
          538 kB

            Assignee:
            ben.caimano@mongodb.com Benjamin Caimano (Inactive)
            Reporter:
            shane.harvey@mongodb.com Shane Harvey
            Votes:
            0 Vote for this issue
            Watchers:
            10 Start watching this issue

              Created:
              Updated:
              Resolved: