-
Type: Bug
-
Resolution: Won't Fix
-
Priority: Critical - P2
-
None
-
Affects Version/s: None
-
Component/s: None
Running format stress configurations with ASAN in the 4.2 branch produces occasional use-after-free errors.
Here's the ASAN report:
==26062==ERROR: AddressSanitizer: heap-use-after-free on address 0x606000a6db70 at pc 0x000000ae02f6 bp 0x7f0db39abaf0 sp 0x7f0db39abae8 READ of size 1 at 0x606000a6db70 thread T17 #0 0xae02f5 in __wt_ref_info /home/ubuntu/src/4.2/build_posix/../src/include/btree.i:1089:24 #1 0xadfd5c in __ref_is_leaf /home/ubuntu/src/4.2/build_posix/../src/btree/bt_walk.c:90:5 #2 0xad906f in __tree_walk_skip_count_callback /home/ubuntu/src/4.2/build_posix/../src/btree/bt_walk.c:598:35 #3 0xad82a3 in __tree_walk_internal /home/ubuntu/src/4.2/build_posix/../src/btree/bt_walk.c:473:17 #4 0xad8df5 in __wt_tree_walk_skip /home/ubuntu/src/4.2/build_posix/../src/btree/bt_walk.c:622:9 #5 0xa21fae in __wt_btcur_next_random /home/ubuntu/src/4.2/build_posix/../src/btree/bt_random.c:581:9 #6 0x60ae6f in __wt_curfile_next_random /home/ubuntu/src/4.2/build_posix/../src/cursor/cur_file.c:120:5 #7 0x5204b6 in random_kv /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/random.c:73:27 #8 0x7f0dbcfb86da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da) #9 0x7f0dbbe6f71e in clone /build/glibc-S9d2JN/glibc-2.27/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95 0x606000a6db70 is located 48 bytes inside of 56-byte region [0x606000a6db40,0x606000a6db78) freed by thread T19 here: #0 0x4c3502 in free /data/mci/7cb6c3b42484980c0d68ddf179e14841/toolchain-builder/tmp/build-llvm.sh-6rk/llvm/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:124:3 #1 0x7334e8 in __wt_free_int /home/ubuntu/src/4.2/build_posix/../src/os_common/os_alloc.c:303:5 #2 0x7b0452 in __wt_ref_addr_free /home/ubuntu/src/4.2/build_posix/../src/include/btree.i:657:9 #3 0x7ae621 in __wt_ref_block_free /home/ubuntu/src/4.2/build_posix/../src/include/btree.i:1131:5 #4 0x79d6c8 in __rec_write_wrapup /home/ubuntu/src/4.2/build_posix/../src/reconcile/rec_write.c:2226:9 #5 0x78d221 in __reconcile /home/ubuntu/src/4.2/build_posix/../src/reconcile/rec_write.c:216:28 #6 0x78bae5 in __wt_reconcile /home/ubuntu/src/4.2/build_posix/../src/reconcile/rec_write.c:103:11 #7 0xabdc87 in __wt_sync_file /home/ubuntu/src/4.2/build_posix/../src/btree/bt_sync.c:311:13 #8 0x8cb0a5 in __checkpoint_tree /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:1626:9 #9 0x8d4582 in __checkpoint_tree_helper /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:1734:11 #10 0x8d433c in __checkpoint_apply /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:197:9 #11 0x8cdfb8 in __txn_checkpoint /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:848:5 #12 0x8c9275 in __txn_checkpoint_wrapper /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:1041:11 #13 0x8c8b7f in __wt_txn_checkpoint /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:1097:9 #14 0x81fbff in __session_checkpoint /home/ubuntu/src/4.2/build_posix/../src/session/session_api.c:1956:11 #15 0x4fd06c in checkpoint /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/checkpoint.c:110:9 #16 0x7f0dbcfb86da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da) previously allocated by thread T4 here: #0 0x4c3a7a in calloc /data/mci/7cb6c3b42484980c0d68ddf179e14841/toolchain-builder/tmp/build-llvm.sh-6rk/llvm/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:155:3 #1 0x731c6a in __wt_calloc /home/ubuntu/src/4.2/build_posix/../src/os_common/os_alloc.c:50:14 #2 0x69cbd0 in __evict_page_dirty_update /home/ubuntu/src/4.2/build_posix/../src/evict/evict_page.c:397:13 #3 0x697771 in __wt_evict /home/ubuntu/src/4.2/build_posix/../src/evict/evict_page.c:196:9 #4 0x677b96 in __evict_page /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:2251:5 #5 0x670846 in __evict_lru_pages /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:1106:20 #6 0x67b5e6 in __evict_pass /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:707:13 #7 0x66fb9c in __evict_server /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:376:5 #8 0x66f308 in __wt_evict_thread_run /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:288:15 #9 0x8a89d3 in __thread_run /home/ubuntu/src/4.2/build_posix/../src/support/thread_group.c:31:9 #10 0x7f0dbcfb86da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da) Thread T17 created by T0 here: #0 0x4ac19d in pthread_create /data/mci/7cb6c3b42484980c0d68ddf179e14841/toolchain-builder/tmp/build-llvm.sh-6rk/llvm/projects/compiler-rt/lib/asan/asan_interceptors.cc:210:3 #1 0x752925 in __wt_thread_create /home/ubuntu/src/4.2/build_posix/../src/os_posix/os_thread.c:28:5 #2 0x50d14b in operations /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/ops.c:203:9 #3 0x52a059 in main /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/t.c:290:13 #4 0x7f0dbbd6fbf6 in __libc_start_main /build/glibc-S9d2JN/glibc-2.27/csu/../csu/libc-start.c:310 Thread T19 created by T0 here: #0 0x4ac19d in pthread_create /data/mci/7cb6c3b42484980c0d68ddf179e14841/toolchain-builder/tmp/build-llvm.sh-6rk/llvm/projects/compiler-rt/lib/asan/asan_interceptors.cc:210:3 #1 0x752925 in __wt_thread_create /home/ubuntu/src/4.2/build_posix/../src/os_posix/os_thread.c:28:5 #2 0x50d3dc in operations /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/ops.c:218:9 #3 0x52a059 in main /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/t.c:290:13 #4 0x7f0dbbd6fbf6 in __libc_start_main /build/glibc-S9d2JN/glibc-2.27/csu/../csu/libc-start.c:310 Thread T4 created by T0 here: #0 0x4ac19d in pthread_create /data/mci/7cb6c3b42484980c0d68ddf179e14841/toolchain-builder/tmp/build-llvm.sh-6rk/llvm/projects/compiler-rt/lib/asan/asan_interceptors.cc:210:3 #1 0x752925 in __wt_thread_create /home/ubuntu/src/4.2/build_posix/../src/os_posix/os_thread.c:28:5 #2 0x8a5ff7 in __thread_group_resize /home/ubuntu/src/4.2/build_posix/../src/support/thread_group.c:204:9 #3 0x8a6b33 in __wt_thread_group_create /home/ubuntu/src/4.2/build_posix/../src/support/thread_group.c:288:5 #4 0x671ebe in __wt_evict_create /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:477:5 #5 0x5dc351 in __wt_connection_workers /home/ubuntu/src/4.2/build_posix/../src/conn/conn_open.c:231:5 #6 0x598454 in wiredtiger_open /home/ubuntu/src/4.2/build_posix/../src/conn/conn_api.c:2720:5 #7 0x534a77 in wts_open /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/wts.c:306:5 #8 0x529bcf in main /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/t.c:280:13 #9 0x7f0dbbd6fbf6 in __libc_start_main /build/glibc-S9d2JN/glibc-2.27/csu/../csu/libc-start.c:310 SUMMARY: AddressSanitizer: heap-use-after-free /home/ubuntu/src/4.2/build_posix/../src/include/btree.i:1089:24 in __wt_ref_info Shadow bytes around the buggy address: 0x0c0c80145b10: fa fa fa fa 00 00 00 00 00 00 00 fa fa fa fa fa 0x0c0c80145b20: 00 00 00 00 00 00 00 fa fa fa fa fa fd fd fd fd 0x0c0c80145b30: fd fd fd fa fa fa fa fa fd fd fd fd fd fd fd fa 0x0c0c80145b40: fa fa fa fa 00 00 00 00 00 00 00 fa fa fa fa fa 0x0c0c80145b50: fd fd fd fd fd fd fd fa fa fa fa fa fa fa fa fa =>0x0c0c80145b60: fa fa fa fa fa fa fa fa fd fd fd fd fd fd[fd]fa 0x0c0c80145b70: fa fa fa fa 00 00 00 00 00 00 00 fa fa fa fa fa 0x0c0c80145b80: fd fd fd fd fd fd fd fa fa fa fa fa 00 00 00 00 0x0c0c80145b90: 00 00 00 fa fa fa fa fa 00 00 00 00 00 00 00 fa 0x0c0c80145ba0: fa fa fa fa 00 00 00 00 00 00 00 00 fa fa fa fa 0x0c0c80145bb0: fd fd fd fd fd fd fd fa fa fa fa fa 00 00 00 00 Shadow byte legend (one shadow byte represents 8 application bytes): Addressable: 00 Partially addressable: 01 02 03 04 05 06 07 Heap left redzone: fa Freed heap region: fd Stack left redzone: f1 Stack mid redzone: f2 Stack right redzone: f3 Stack after return: f5 Stack use after scope: f8 Global redzone: f9 Global init order: f6 Poisoned by user: f7 Container overflow: fc Array cookie: ac Intra object redzone: bb ASan internal: fe Left alloca redzone: ca Right alloca redzone: cb Shadow gap: cc ==26062==ABORTING AddressSanitizer:DEADLYSIGNAL AddressSanitizer:DEADLYSIGNAL AddressSanitizer:DEADLYSIGNAL AddressSanitizer:DEADLYSIGNAL
Here's the failing config:
############################################ # RUN PARAMETERS: V2 ############################################ assert.commit_timestamp=0 assert.read_timestamp=0 backup=0 backup.incremental=off backup.incr_granularity=2120 btree.bitcnt=8 btree.compression=snappy btree.dictionary=0 btree.huffman_key=0 btree.huffman_value=0 btree.internal_key_truncation=1 btree.internal_page_max=13 btree.key_gap=11 btree.key_max=65 btree.key_min=30 btree.leaf_page_max=11 btree.memory_page_max=6 btree.prefix_compression=1 btree.prefix_compression_min=0 btree.repeat_data_pct=79 btree.reverse=0 btree.split_pct=78 btree.value_max=1421 btree.value_min=18 cache=93 cache.evict_max=1 cache.minimum=20 checkpoint=on checkpoint.log_size=36 checkpoint.wait=16 disk.checksum=uncompressed disk.data_extend=0 disk.direct_io=0 disk.encryption=rotn-7 disk.firstfit=0 disk.mmap=1 disk.mmap_all=0 format.abort=0 format.independent_thread_rng=0 format.major_timeout=0 logging=1 logging.archive=0 logging.compression=snappy logging.file_max=216011 logging.prealloc=0 lsm.auto_throttle=1 lsm.bloom=1 lsm.bloom_bit_count=41 lsm.bloom_hash_count=20 lsm.bloom_oldest=0 lsm.chunk_size=10 lsm.merge_max=12 lsm.worker_threads=3 ops.alter=0 ops.compaction=0 ops.pct.delete=30 ops.pct.insert=61 ops.pct.modify=1 ops.pct.read=4 ops.pct.write=4 ops.prepare=0 ops.random_cursor=1 ops.rebalance=1 ops.salvage=1 ops.truncate=1 ops.verify=1 quiet=1 runs=1 runs.in_memory=0 runs.ops=0 runs.rows=1000000 runs.source=file runs.threads=4 runs.timer=6 runs.type=row-store statistics=0 statistics.server=0 stress.aggressive_sweep=0 stress.checkpoint=0 stress.hs_sweep=0 stress.split_1=0 stress.split_2=0 stress.split_3=0 stress.split_4=0 stress.split_5=0 stress.split_6=0 stress.split_7=0 stress.split_8=0 transaction.frequency=100 transaction.isolation=snapshot transaction.timestamps=1 wiredtiger.config= wiredtiger.rwlock=1 wiredtiger.leak_memory=0 ############################################