Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-8690

Checkpoint timeout on an eviction gen

    • Type: Icon: Build Failure Build Failure
    • Resolution: Cannot Reproduce
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: None
    • None
    • 8
    • Storage - Ra 2022-01-24, Storage - Ra 2022-02-07

      The issues comes from BF-24056. The error shows checkpoint times out on an eviction gen:

      [j0:sec1] {"t":{"$date":"2022-01-14T16:45:35.029+00:00"},"s":"I",  "c":"WT",       "id":22430,   "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1642178735,"ts_usec":14637,"thread":"5228:140733066072672","session_dhandle_name":"file:index-8567-1962825563001122622.wt","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_GENERATION","category_id":16,"verbose_level":"NOTICE","verbose_level_id":-1,"msg":"evict generation drain waited 3 minutes"}}}
      [j0:sec1] {"t":{"$date":"2022-01-14T16:45:35.047+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"Checkpointer","msg":"WiredTiger error message","attr":{"error":0,"message":{"ts_sec":1642178735,"ts_usec":39634,"thread":"5228:140733066072672","session_dhandle_name":"file:index-8567-1962825563001122622.wt","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"void __cdecl __wt_gen_drain(struct __wt_session_impl *,int,unsigned __int64):169:minutes < 4"}}}
      [j0:sec1] {"t":{"$date":"2022-01-14T16:45:35.059+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"Checkpointer","msg":"WiredTiger error message","attr":{"error":0,"message":{"ts_sec":1642178735,"ts_usec":49634,"thread":"5228:140733066072672","session_dhandle_name":"file:index-8567-1962825563001122622.wt","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"void __cdecl __wt_abort(struct __wt_session_impl *):28:aborting WiredTiger library"}}}

      Stacktrace:

      {"a":"7FF7356497FC","module":"mongod.exe","file":".../src/third_party/wiredtiger/src/os_common/os_abort.c","line":30,"s":"__wt_abort","s+":"2C"},
      {"a":"7FF7356AD947","module":"mongod.exe","file":".../src/third_party/wiredtiger/src/support/generation.c","line":130,"s":"__wt_gen_drain","s+":"247"},
      {"a":"7FF73571FF4C","module":"mongod.exe","file":".../src/third_party/wiredtiger/src/btree/bt_sync.c","line":537,"s":"__wt_sync_file","s+":"2DC"},
      {"a":"7FF7356A73F6","module":"mongod.exe","file":".../src/third_party/wiredtiger/src/txn/txn_ckpt.c","line":1878,"s":"__checkpoint_tree","s+":"1B6"},
      {"a":"7FF7356A75C2","module":"mongod.exe","file":".../src/third_party/wiredtiger/src/txn/txn_ckpt.c","line":1994,"s":"__checkpoint_tree_helper","s+":"62"},
      {"a":"7FF7356A5086","module":"mongod.exe","file":".../src/third_party/wiredtiger/src/txn/txn_ckpt.c","line":198,"s":"__checkpoint_apply_to_dhandles","s+":"56"},
      {"a":"7FF7356A8365","module":"mongod.exe","file":".../src/third_party/wiredtiger/src/txn/txn_ckpt.c","line":920,"s":"__txn_checkpoint","s+":"585"},
      {"a":"7FF7356A8CE9","module":"mongod.exe","file":".../src/third_party/wiredtiger/src/txn/txn_ckpt.c","line":1165,"s":"__txn_checkpoint_wrapper","s+":"189"},
      {"a":"7FF7356A9B33","module":"mongod.exe","file":".../src/third_party/wiredtiger/src/txn/txn_ckpt.c","line":1221,"s":"__wt_txn_checkpoint","s+":"113"},
      {"a":"7FF7355EDD2C","module":"mongod.exe","file":".../src/third_party/wiredtiger/src/session/session_api.c","line":1916,"s":"__session_checkpoint","s+":"4DC"},
      {"a":"7FF733E2EB22","module":"mongod.exe","file":".../src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp","line":1917,"s":"mongo::WiredTigerKVEngine::checkpoint","s+":"A42"},

        1. all_nodes.png
          all_nodes.png
          734 kB
        2. all_stats.zip
          6.27 MB
        3. checkpoint_stack.png
          checkpoint_stack.png
          496 kB
        4. evergreen_backtrace.txt
          97 kB
        5. hs_update_stats.png
          hs_update_stats.png
          105 kB
        6. metrics.2022-01-14T15-41-30Z-00000
          1.79 MB
        7. metrics.interim
          43 kB
        8. node_2_stats.png
          node_2_stats.png
          300 kB
        9. row_search_stack.png
          row_search_stack.png
          601 kB
        10. wt_threads.png
          wt_threads.png
          794 kB

            Assignee:
            etienne.petrel@mongodb.com Etienne Petrel
            Reporter:
            etienne.petrel@mongodb.com Etienne Petrel
            Votes:
            0 Vote for this issue
            Watchers:
            10 Start watching this issue

              Created:
              Updated:
              Resolved: