Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-6118

Fix missing checkpoint in backup

    • 8
    • Storage - Tora 2020-05-18
    • v4.4, v4.2, v4.0

      Running test/format on my local machine hit a CONFIG that gives a read checksum error during recovery of the application table:

      [1588603795:516511][18209:0x7f039e7fc700], t, file:wt.wt, txn-recover: __wt_block_read_off, 283: wt.wt: read checksum error for 1536B block at offset 6468608: block header checksum of 0x109dfedb doesn't match expected checksum of 0xb9c5ff81
      

      This mostly reproduces, but not 100% of the time. A rerun of the CONFIG hit on iteration 2, but iteration 1 succeeded.

      Here's the stack:

      (gdb) bt
      #0  0x00007f03d09405f7 in raise () from /lib64/libc.so.6
      #1  0x00007f03d0941ce8 in abort () from /lib64/libc.so.6
      #2  0x000000000049a15d in __wt_abort (session=0x7f033c0291e0) at ../src/os_common/os_abort.c:30
      #3  0x00000000004e1972 in __wt_panic_func (session=0x7f033c0291e0, error=-31802, 
          func=0x643060 <__func__.18293> "__wt_block_read_off", line=292, fmt=0x642fd1 "%s: fatal read error") at ../src/support/err.c:378
      #4  0x0000000000529007 in __wt_block_read_off (session=0x7f033c0291e0, block=0x7f033e081210, buf=0x7f033c05e9e0, offset=6468608, 
          size=1536, checksum=3116760961) at ../src/block/block_read.c:292
      #5  0x0000000000523236 in __wt_block_extlist_read (session=0x7f033c0291e0, block=0x7f033e081210, el=0x7f033e0813c0, 
          ckpt_size=6470144) at ../src/block/block_ext.c:1098
      #6  0x0000000000523125 in __wt_block_extlist_read_avail (session=0x7f033c0291e0, block=0x7f033e081210, el=0x7f033e0813c0, 
          ckpt_size=6470144) at ../src/block/block_ext.c:1062
      #7  0x000000000051e237 in __wt_block_checkpoint_load (session=0x7f033c0291e0, block=0x7f033e081210, 
          addr=0x7f033e0801d0 "\001\342\021\027\201\344*\330Md\342\021\030\201\344\333J\317C\342\021\031\203\344\271\305\337A\200\200\200\343b\231\300\343b\215\300", addr_size=39, root_addr=0x7f039e7fac00 "\342\021\027\201\344*\330Md\334d", 
          root_addr_sizep=0x7f039e7fad00, checkpoint=false) at ../src/block/block_ckpt.c:107
      #8  0x00000000005243ad in __bm_checkpoint_load (bm=0x7f033e07f680, session=0x7f033c0291e0, 
          addr=0x7f033e0801d0 "\001\342\021\027\201\344*\330Md\342\021\030\201\344\333J\317C\342\021\031\203\344\271\305\337A\200\200\200\343b\231\300\343b\215\300", addr_size=39, root_addr=0x7f039e7fac00 "\342\021\027\201\344*\330Md\334d", 
          root_addr_sizep=0x7f039e7fad00, checkpoint=false) at ../src/block/block_mgr.c:103
      #9  0x00000000005527ac in __wt_btree_open (session=0x7f033c0291e0, op_cfg=0x7f039e7fb220) at ../src/btree/bt_handle.c:145
      #10 0x000000000043b812 in __wt_conn_dhandle_open (session=0x7f033c0291e0, cfg=0x7f039e7fb220, flags=0)
          at ../src/conn/conn_dhandle.c:465
      #11 0x00000000004dfbfd in __wt_session_get_dhandle (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", checkpoint=0x0, 
          cfg=0x7f039e7fb220, flags=0) at ../src/session/session_dhandle.c:511
      #12 0x00000000004dfba6 in __wt_session_get_dhandle (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", checkpoint=0x0, 
          cfg=0x7f039e7fb220, flags=0) at ../src/session/session_dhandle.c:504
      #13 0x00000000004df25f in __wt_session_get_btree_ckpt (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", cfg=0x7f039e7fb220, 
          flags=0) at ../src/session/session_dhandle.c:320
      #14 0x0000000000452260 in __wt_curfile_open (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", owner=0x0, 
          cfg=0x7f039e7fb220, cursorp=0x7f039e7fb238) at ../src/cursor/cur_file.c:801
      #15 0x00000000004ca6f5 in __session_open_cursor_int (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", owner=0x0, other=0x0, 
          cfg=0x7f039e7fb220, cursorp=0x7f039e7fb238) at ../src/session/session_api.c:462
      #16 0x00000000004caa43 in __wt_open_cursor (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", owner=0x0, cfg=0x7f039e7fb220, 
          cursorp=0x7f039e7fb238) at ../src/session/session_api.c:526
      #17 0x00000000005085ff in __recovery_cursor (session=0x7f033c0291e0, r=0x7f039e7fb6b0, lsnp=0x7f039e7fb560, id=2, duplicate=false, 
          cp=0x7f039e7fb3e0) at ../src/txn/txn_recover.c:79
      #18 0x000000000050921a in __txn_op_apply (r=0x7f039e7fb6b0, lsnp=0x7f039e7fb560, pp=0x7f039e7fb490, end=0x7f033e08022f "")
          at ../src/txn/txn_recover.c:223
      #19 0x0000000000509737 in __txn_commit_apply (r=0x7f039e7fb6b0, lsnp=0x7f039e7fb560, pp=0x7f039e7fb490, 
          end=0x7f033e080280 "tent),block_allocation=best,block_compressor=\"zlib\",cache_resident=false,checksum=\"on\",collator=,columns=,dictionary=0,encryption=(keyid=,name=),format=btree,huffman_key=,huffman_value=,id=2,ignore_in"...) at ../src/txn/txn_recover.c:298
      #20 0x00000000005098e9 in __txn_log_recover (session=0x7f033c0291e0, logrec=0x7f033c05d3e0, lsnp=0x7f039e7fb560, 
          next_lsnp=0x7f039e7fb590, cookie=0x7f039e7fb6b0, firstrecord=0) at ../src/txn/txn_recover.c:343
      #21 0x000000000047e13f in __wt_log_scan (session=0x7f033c0291e0, lsnp=0x0, flags=9, func=0x509767 <__txn_log_recover>, 
          cookie=0x7f039e7fb6b0) at ../src/log/log.c:2387
      #22 0x000000000050a808 in __wt_txn_recover (session=0x7f033c0291e0) at ../src/txn/txn_recover.c:690
      #23 0x000000000043e9b4 in __wt_connection_workers (session=0x7f033c028a10, cfg=0x7f039e7fb7e0) at ../src/conn/conn_open.c:215
      #24 0x00000000004382b2 in wiredtiger_open (home=0x7f033c006be0 "RUNDIR/BACKUP", event_handler=0x89dea0 <event_handler>, 
      

            Assignee:
            keith.smith@mongodb.com Keith Smith
            Reporter:
            sue.loverso@mongodb.com Susan LoVerso
            Votes:
            0 Vote for this issue
            Watchers:
            8 Start watching this issue

              Created:
              Updated:
              Resolved: