New test failure:
http://build.wiredtiger.com:8080/job/wiredtiger-test-format-stress-zseries/38478/console
file:wt.wt, WT_SESSION.checkpoint: __rec_txn_read, 1350: upd->type != WT_UPDATE_BIRTHMARK file:wt.wt, WT_SESSION.checkpoint: aborting WiredTiger library Backtrace: /lib64/libc.so.6(gsignal+0x40)[0x3ff84640a78] /lib64/libc.so.6(abort+0x128)[0x3ff84642300] ??:?(__wt_abort)[0x80063056] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/support/err.c:501(__wt_assert)[0x800ba282] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/reconcile/rec_write.c:1351(__rec_txn_read)[0x8008360a] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/reconcile/rec_write.c:5382(__rec_row_leaf)[0x8008da76] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/reconcile/rec_write.c:474(__wt_reconcile)[0x80080fb4] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/evict/evict_page.c:597(__evict_review)[0x8004ebb0] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/evict/evict_page.c:144(__wt_evict)[0x8004d79e] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/evict/evict_page.c:85(__wt_page_release_evict)[0x8004d298] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/btree/bt_sync.c:298(__sync_file)[0x80132df2] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/btree/bt_sync.c:401(__wt_cache_op)[0x801332be] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/txn/txn_ckpt.c:1564(__checkpoint_tree)[0x800d9df4] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/txn/txn_ckpt.c:1688(__checkpoint_tree_helper)[0x800da270] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/txn/txn_ckpt.c:217(__checkpoint_apply)[0x800d62d4] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/txn/txn_ckpt.c:861(__txn_checkpoint)[0x800d7e66] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/txn/txn_ckpt.c:1036(__txn_checkpoint_wrapper)[0x800d889a] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/txn/txn_ckpt.c:1100 (discriminator 2)(__wt_txn_checkpoint)[0x800d8a6a] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/session/session_api.c:1703(__session_checkpoint)[0x800b27e4] /mnt/workspace/wiredtiger-test-format-stress-zseries/build_posix/../src/conn/conn_ckpt.c:116(__ckpt_server)[0x8015db28] /lib64/libpthread.so.0(+0x81f2)[0x3ff849881f2] /lib64/libc.so.6(+0x1098da)[0x3ff847098da]
Here's the CONFIG:
############################################ # RUN PARAMETERS ############################################ abort=0 alter=0 auto_throttle=1 backups=0 bitcnt=7 bloom=1 bloom_bit_count=9 bloom_hash_count=14 bloom_oldest=0 cache=59 cache_minimum=20 checkpoints=wiredtiger checkpoint_log_size=49 checkpoint_wait=25 checksum=uncompressed chunk_size=9 compaction=0 compression=lz4-noraw data_extend=0 data_source=table delete_pct=12 dictionary=0 direct_io=0 encryption=none evict_max=3 file_type=row-store firstfit=1 huffman_key=0 huffman_value=0 independent_thread_rng=1 in_memory=0 insert_pct=28 internal_key_truncation=1 internal_page_max=16 isolation=read-committed key_gap=19 key_max=63 key_min=24 leaf_page_max=14 leak_memory=0 logging=0 logging_archive=1 logging_compression=none logging_file_max=85520 logging_prealloc=1 long_running_txn=0 lsm_worker_threads=4 merge_max=12 mmap=1 modify_pct=53 ops=100000 prefix_compression=1 prefix_compression_min=8 quiet=1 read_pct=5 rebalance=1 repeat_data_pct=30 reverse=0 rows=100000 runs=100 salvage=1 split_pct=70 statistics=0 statistics_server=0 threads=16 timer=360 transaction_timestamps=1 transaction-frequency=1 value_max=3621 value_min=17 verify=1 wiredtiger_config= write_pct=2 ############################################
Here's the stack:
#0 0x000003ff84640a78 in raise () from /lib64/libc.so.6 #1 0x000003ff84642300 in abort () from /lib64/libc.so.6 #2 0x0000000080063056 in __wt_abort (session=0x9236d5d0) at ../src/os_common/os_abort.c:31 #3 0x00000000800ba282 in __wt_assert (session=0x9236d5d0, error=0, file_name=0x8021d7da <__func__.14016> "__rec_txn_read", line_number=1350, fmt=0x8021ca70 "%s") at ../src/support/err.c:501 #4 0x000000008008360a in __rec_txn_read (session=0x9236d5d0, r=0x3ff547341c0, ins=0x0, ripcip=0x3ff547fd250, vpack=0x3ff6d7fe168, upd_savedp=0x0, updp=0x3ff6d7fe148) at ../src/reconcile/rec_write.c:1350 #5 0x000000008008da76 in __rec_row_leaf (session=0x9236d5d0, r=0x3ff547341c0, page=0x3ff547fd100, salvage=0x0) at ../src/reconcile/rec_write.c:5382 #6 0x0000000080080fb4 in __wt_reconcile (session=0x9236d5d0, ref=0x9248fa80, salvage=0x0, flags=10, lookaside_retryp=0x0) at ../src/reconcile/rec_write.c:474 #7 0x000000008004ebb0 in __evict_review (session=0x9236d5d0, ref=0x9248fa80, closing=false, inmem_splitp=0x3ff6d7fe55d) at ../src/evict/evict_page.c:597 #8 0x000000008004d79e in __wt_evict (session=0x9236d5d0, ref=0x9248fa80, closing=false) at ../src/evict/evict_page.c:144 #9 0x000000008004d298 in __wt_page_release_evict (session=0x9236d5d0, ref=0x9248fa80) at ../src/evict/evict_page.c:85 #10 0x0000000080132df2 in __sync_file (session=0x9236d5d0, syncop=WT_SYNC_CHECKPOINT) at ../src/btree/bt_sync.c:298 #11 0x00000000801332be in __wt_cache_op (session=0x9236d5d0, op=WT_SYNC_CHECKPOINT) at ../src/btree/bt_sync.c:401 #12 0x00000000800d9df4 in __checkpoint_tree (session=0x9236d5d0, is_checkpoint=true, cfg=0x3ff6d7feed8) at ../src/txn/txn_ckpt.c:1564 #13 0x00000000800da270 in __checkpoint_tree_helper (session=0x9236d5d0, cfg=0x3ff6d7feed8) at ../src/txn/txn_ckpt.c:1688 #14 0x00000000800d62d4 in __checkpoint_apply (session=0x9236d5d0, cfg=0x3ff6d7feed8, op=0x800da1c0 <__checkpoint_tree_helper>) at ../src/txn/txn_ckpt.c:217 #15 0x00000000800d7e66 in __txn_checkpoint (session=0x9236d5d0, cfg=0x3ff6d7feed8) at ../src/txn/txn_ckpt.c:861 #16 0x00000000800d889a in __txn_checkpoint_wrapper (session=0x9236d5d0, cfg=0x3ff6d7feed8) at ../src/txn/txn_ckpt.c:1036 #17 0x00000000800d8a6a in __wt_txn_checkpoint (session=0x9236d5d0, cfg=0x3ff6d7feed8, waiting=true) at ../src/txn/txn_ckpt.c:1100 #18 0x00000000800b27e4 in __session_checkpoint (wt_session=0x9236d5d0, config=0x0) at ../src/session/session_api.c:1703 #19 0x000000008015db28 in __ckpt_server (arg=0x9236d5d0) at ../src/conn/conn_ckpt.c:116