-
Type: Bug
-
Resolution: Fixed
-
Priority: Major - P3
-
Affects Version/s: None
-
Component/s: None
-
8
-
Storage - Tora 2020-05-18
-
v4.4, v4.2, v4.0
Running test/format on my local machine hit a CONFIG that gives a read checksum error during recovery of the application table:
[1588603795:516511][18209:0x7f039e7fc700], t, file:wt.wt, txn-recover: __wt_block_read_off, 283: wt.wt: read checksum error for 1536B block at offset 6468608: block header checksum of 0x109dfedb doesn't match expected checksum of 0xb9c5ff81
This mostly reproduces, but not 100% of the time. A rerun of the CONFIG hit on iteration 2, but iteration 1 succeeded.
Here's the stack:
(gdb) bt #0 0x00007f03d09405f7 in raise () from /lib64/libc.so.6 #1 0x00007f03d0941ce8 in abort () from /lib64/libc.so.6 #2 0x000000000049a15d in __wt_abort (session=0x7f033c0291e0) at ../src/os_common/os_abort.c:30 #3 0x00000000004e1972 in __wt_panic_func (session=0x7f033c0291e0, error=-31802, func=0x643060 <__func__.18293> "__wt_block_read_off", line=292, fmt=0x642fd1 "%s: fatal read error") at ../src/support/err.c:378 #4 0x0000000000529007 in __wt_block_read_off (session=0x7f033c0291e0, block=0x7f033e081210, buf=0x7f033c05e9e0, offset=6468608, size=1536, checksum=3116760961) at ../src/block/block_read.c:292 #5 0x0000000000523236 in __wt_block_extlist_read (session=0x7f033c0291e0, block=0x7f033e081210, el=0x7f033e0813c0, ckpt_size=6470144) at ../src/block/block_ext.c:1098 #6 0x0000000000523125 in __wt_block_extlist_read_avail (session=0x7f033c0291e0, block=0x7f033e081210, el=0x7f033e0813c0, ckpt_size=6470144) at ../src/block/block_ext.c:1062 #7 0x000000000051e237 in __wt_block_checkpoint_load (session=0x7f033c0291e0, block=0x7f033e081210, addr=0x7f033e0801d0 "\001\342\021\027\201\344*\330Md\342\021\030\201\344\333J\317C\342\021\031\203\344\271\305\337A\200\200\200\343b\231\300\343b\215\300", addr_size=39, root_addr=0x7f039e7fac00 "\342\021\027\201\344*\330Md\334d", root_addr_sizep=0x7f039e7fad00, checkpoint=false) at ../src/block/block_ckpt.c:107 #8 0x00000000005243ad in __bm_checkpoint_load (bm=0x7f033e07f680, session=0x7f033c0291e0, addr=0x7f033e0801d0 "\001\342\021\027\201\344*\330Md\342\021\030\201\344\333J\317C\342\021\031\203\344\271\305\337A\200\200\200\343b\231\300\343b\215\300", addr_size=39, root_addr=0x7f039e7fac00 "\342\021\027\201\344*\330Md\334d", root_addr_sizep=0x7f039e7fad00, checkpoint=false) at ../src/block/block_mgr.c:103 #9 0x00000000005527ac in __wt_btree_open (session=0x7f033c0291e0, op_cfg=0x7f039e7fb220) at ../src/btree/bt_handle.c:145 #10 0x000000000043b812 in __wt_conn_dhandle_open (session=0x7f033c0291e0, cfg=0x7f039e7fb220, flags=0) at ../src/conn/conn_dhandle.c:465 #11 0x00000000004dfbfd in __wt_session_get_dhandle (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", checkpoint=0x0, cfg=0x7f039e7fb220, flags=0) at ../src/session/session_dhandle.c:511 #12 0x00000000004dfba6 in __wt_session_get_dhandle (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", checkpoint=0x0, cfg=0x7f039e7fb220, flags=0) at ../src/session/session_dhandle.c:504 #13 0x00000000004df25f in __wt_session_get_btree_ckpt (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", cfg=0x7f039e7fb220, flags=0) at ../src/session/session_dhandle.c:320 #14 0x0000000000452260 in __wt_curfile_open (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", owner=0x0, cfg=0x7f039e7fb220, cursorp=0x7f039e7fb238) at ../src/cursor/cur_file.c:801 #15 0x00000000004ca6f5 in __session_open_cursor_int (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", owner=0x0, other=0x0, cfg=0x7f039e7fb220, cursorp=0x7f039e7fb238) at ../src/session/session_api.c:462 #16 0x00000000004caa43 in __wt_open_cursor (session=0x7f033c0291e0, uri=0x7f033c066fe0 "file:wt.wt", owner=0x0, cfg=0x7f039e7fb220, cursorp=0x7f039e7fb238) at ../src/session/session_api.c:526 #17 0x00000000005085ff in __recovery_cursor (session=0x7f033c0291e0, r=0x7f039e7fb6b0, lsnp=0x7f039e7fb560, id=2, duplicate=false, cp=0x7f039e7fb3e0) at ../src/txn/txn_recover.c:79 #18 0x000000000050921a in __txn_op_apply (r=0x7f039e7fb6b0, lsnp=0x7f039e7fb560, pp=0x7f039e7fb490, end=0x7f033e08022f "") at ../src/txn/txn_recover.c:223 #19 0x0000000000509737 in __txn_commit_apply (r=0x7f039e7fb6b0, lsnp=0x7f039e7fb560, pp=0x7f039e7fb490, end=0x7f033e080280 "tent),block_allocation=best,block_compressor=\"zlib\",cache_resident=false,checksum=\"on\",collator=,columns=,dictionary=0,encryption=(keyid=,name=),format=btree,huffman_key=,huffman_value=,id=2,ignore_in"...) at ../src/txn/txn_recover.c:298 #20 0x00000000005098e9 in __txn_log_recover (session=0x7f033c0291e0, logrec=0x7f033c05d3e0, lsnp=0x7f039e7fb560, next_lsnp=0x7f039e7fb590, cookie=0x7f039e7fb6b0, firstrecord=0) at ../src/txn/txn_recover.c:343 #21 0x000000000047e13f in __wt_log_scan (session=0x7f033c0291e0, lsnp=0x0, flags=9, func=0x509767 <__txn_log_recover>, cookie=0x7f039e7fb6b0) at ../src/log/log.c:2387 #22 0x000000000050a808 in __wt_txn_recover (session=0x7f033c0291e0) at ../src/txn/txn_recover.c:690 #23 0x000000000043e9b4 in __wt_connection_workers (session=0x7f033c028a10, cfg=0x7f039e7fb7e0) at ../src/conn/conn_open.c:215 #24 0x00000000004382b2 in wiredtiger_open (home=0x7f033c006be0 "RUNDIR/BACKUP", event_handler=0x89dea0 <event_handler>,