-
Type: Bug
-
Resolution: Fixed
-
Priority: Major - P3
-
Affects Version/s: None
-
Component/s: None
-
None
-
5
-
Storage - Ra 2020-06-29, Storage - Ra 2020-07-13
We've seen an issue where we hit a segfault when walking an update chain for a history store key. The pointer that we try to dereference has a magic value 0xcdcdcdcdcdcdcdcd which is something that TCMalloc uses when deallocating (I don't know the specifics but there is a code chunk showing this magic here).
I think the fact that we ignore tombstones in history store is suspicious because it may cause us to look past an update that we usually wouldn't. I do see that we return a tombstone if it is globally visible, however (which means we're not going to be looking past globally visible updates).
GDB output:
(gdb) bt #0 __wt_txn_read_upd_list (session=0x7fedf77a97c0, cbt=0x7fedf4e18da0, upd=0xcdcdcdcdcdcdcdcd, prepare_updp=0x0) at src/third_party/wiredtiger/src/include/txn.i:849 #1 0x00007fee00c7288e in __cursor_row_prev (skippedp=<synthetic pointer>, restart=<optimized out>, newpage=<optimized out>, cbt=0x7fedf4e18da0) at src/third_party/wiredtiger/src/btree/bt_curprev.c:517 #2 __wt_btcur_prev (cbt=cbt@entry=0x7fedf4e18da0, truncating=truncating@entry=false) at src/third_party/wiredtiger/src/btree/bt_curprev.c:657 #3 0x00007fee00c7c0aa in __wt_btcur_search_near (cbt=cbt@entry=0x7fedf4e18da0, exactp=exactp@entry=0x7fedf1f1c174) at src/third_party/wiredtiger/src/btree/bt_cursor.c:691 #4 0x00007fee00bc5d00 in __curfile_search_near (cursor=0x7fedf4e18da0, exact=0x7fedf1f1c174) at src/third_party/wiredtiger/src/cursor/cur_file.c:231 #5 0x00007fee00be2a5e in __hs_delete_key_from_ts_int (ts=0, key=0x7fedc0437c80, btree_id=46, session=0x7fedf77a97c0) at src/third_party/wiredtiger/src/history/hs.c:1313 #6 __wt_hs_delete_key_from_ts (session=session@entry=0x7fedf77a97c0, btree_id=46, key=0x7fedc0437c80, ts=ts@entry=0) at src/third_party/wiredtiger/src/history/hs.c:1378 #7 0x00007fee00d39ed3 in __wt_rec_row_leaf (session=session@entry=0x7fedf77a97c0, r=r@entry=0x7fedc2eed420, pageref=pageref@entry=0x7fedc1e53fe0, salvage=salvage@entry=0x0) at src/third_party/wiredtiger/src/reconcile/rec_row.c:871 #8 0x00007fee00c15fb5 in __reconcile (page_lockedp=<synthetic pointer>, flags=172, salvage=0x0, ref=0x7fedc1e53fe0, session=0x7fedf77a97c0) at src/third_party/wiredtiger/src/reconcile/rec_write.c:182 #9 __wt_reconcile (session=session@entry=0x7fedf77a97c0, ref=ref@entry=0x7fedc1e53fe0, salvage=salvage@entry=0x0, flags=flags@entry=172) at src/third_party/wiredtiger/src/reconcile/rec_write.c:89 #10 0x00007fee00bdda5f in __evict_review (inmem_splitp=<synthetic pointer>, evict_flags=0, ref=0x7fedc1e53fe0, session=0x7fedf77a97c0) at src/third_party/wiredtiger/src/evict/evict_page.c:658 #11 __wt_evict (session=session@entry=0x7fedf77a97c0, ref=ref@entry=0x7fedc1e53fe0, previous_state=previous_state@entry=3 '\003', flags=flags@entry=0) at src/third_party/wiredtiger/src/evict/evict_page.c:186 #12 0x00007fee00bd5e19 in __evict_page (session=0x7fedf77a97c0, is_server=false) at src/third_party/wiredtiger/src/evict/evict_lru.c:2254 #13 0x00007fee00bd67f5 in __evict_lru_pages (session=session@entry=0x7fedf77a97c0, is_server=is_server@entry=false) at src/third_party/wiredtiger/src/evict/evict_lru.c:1134 #14 0x00007fee00bd8be4 in __wt_evict_thread_run (session=0x7fedf77a97c0, thread=0x7fedfc30c420) at src/third_party/wiredtiger/src/evict/evict_lru.c:324 #15 0x00007fee00c44c89 in __thread_run (arg=0x7fedfc30c420) at src/third_party/wiredtiger/src/support/thread_group.c:31 #16 0x00007fedfe3a7aa1 in start_thread () from /lib64/libpthread.so.0 #17 0x00007fedfe0f4c4d in clone () from /lib64/libc.so.6 (gdb) f 0 #0 __wt_txn_read_upd_list (session=0x7fedf77a97c0, cbt=0x7fedf4e18da0, upd=0xcdcdcdcdcdcdcdcd, prepare_updp=0x0) at src/third_party/wiredtiger/src/include/txn.i:849 849 WT_ORDERED_READ(type, upd->type); (gdb) p upd $1 = (WT_UPDATE *) 0xcdcdcdcdcdcdcdcd (gdb) p cbt->ins->upd $2 = (WT_UPDATE *) 0x7fedc35c54c0 (gdb) p *cbt->ins->upd $3 = {txnid = 133879, durable_ts = 0, start_ts = 0, next = 0x7fedc17dd3e0, size = 0, type = 4 '\004', prepare_state = 0 '\000', flags = 0 '\000', data = 0x7fedc35c54e7 "'"} (gdb) p *cbt->ins->upd->next $4 = {txnid = 133862, durable_ts = 0, start_ts = 0, next = 0x0, size = 74, type = 3 '\003', prepare_state = 0 '\000', flags = 0 '\000', data = 0x7fedc17dd407 "\200\200\203G"}