Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-3835

cursor remove tries to return a key that doesn't exist

    • Type: Icon: Bug Bug
    • Resolution: Fixed
    • Priority: Icon: Major - P3 Major - P3
    • 3.6.3, 3.7.2, WT3.1.0
    • Affects Version/s: None
    • Component/s: None
    • None
    • Storage 2018-01-15, Storage 2018-01-29

      The Jenkins job failed for wiredtiger-test-race-condition-stress-sanitizer with AddressSanitizer complaining of heap-buffer-overflow:
      http://build.wiredtiger.com:8080/job/wiredtiger-test-race-condition-stress-sanitizer/15116/console

      ############################################
      #  RUN PARAMETERS
      ############################################
      abort=0
      alter=0
      auto_throttle=1
      backups=0
      bitcnt=4
      bloom=1
      bloom_bit_count=43
      bloom_hash_count=16
      bloom_oldest=0
      cache=37
      cache_minimum=20
      checkpoints=off
      checkpoint_log_size=21
      checkpoint_wait=86
      checksum=uncompressed
      chunk_size=8
      compaction=0
      compression=none
      data_extend=0
      data_source=file
      delete_pct=80
      dictionary=0
      direct_io=0
      encryption=none
      evict_max=3
      file_type=row-store
      firstfit=0
      huffman_key=0
      huffman_value=0
      independent_thread_rng=1
      in_memory=1
      insert_pct=1
      internal_key_truncation=1
      internal_page_max=15
      isolation=random
      key_gap=10
      key_max=32
      key_min=29
      leaf_page_max=9
      leak_memory=0
      logging=0
      logging_archive=1
      logging_compression=none
      logging_file_max=12507
      logging_prealloc=1
      long_running_txn=0
      lsm_worker_threads=3
      merge_max=4
      mmap=1
      modify_pct=2
      ops=100000
      prefix_compression=1
      prefix_compression_min=6
      quiet=1
      read_pct=2
      rebalance=0
      repeat_data_pct=68
      reverse=0
      rows=100000
      runs=1
      salvage=0
      split_pct=54
      statistics=0
      statistics_server=0
      threads=27
      timer=360
      transaction_timestamps=0
      transaction-frequency=62
      value_max=80
      value_min=3
      verify=0
      wiredtiger_config=
      write_pct=15
      ############################################
      

      The Error:

      ==40111==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x60700015a790 at pc 0x000000869ff5 bp 0x7f6bb2778cc0 sp 0x7f6bb2778cb8
      READ of size 8 at 0x60700015a790 thread T58
          #0 0x869ff4 in __wt_row_leaf_key /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/include/btree.i:1001:9
          #1 0x865177 in __key_return /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/btree/bt_ret.c:62:11
          #2 0x8649ff in __wt_key_return /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/btree/bt_ret.c:267:3
          #3 0xaf6da7 in __wt_btcur_remove /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:1089:4
          #4 0x97dffa in __curfile_remove /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/cursor/cur_file.c:398:2
          #5 0x52c2a3 in row_remove /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:1698:9
          #6 0x527196 in ops /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:808:9
          #7 0x7f6bc7067dc4 in start_thread /usr/src/debug/glibc-2.17-c758a686/nptl/pthread_create.c:308
          #8 0x7f6bc624e76c in __clone (/lib64/libc.so.6+0xf776c)
      
      0x60700015a790 is located 0 bytes to the right of 80-byte region [0x60700015a740,0x60700015a790)
      allocated by thread T58 here:
          #0 0x4dfa6d in calloc /home/bostic/src/llvm40/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:74
          #1 0x6330b2 in __wt_calloc /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/os_common/os_alloc.c:52:11
          #2 0x836ee9 in __wt_page_alloc /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/btree/bt_page.c:63:2
          #3 0x81ae8c in __wt_btree_new_leaf_page /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/btree/bt_handle.c:694:3
          #4 0x84c94c in __page_read /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/btree/bt_read.c:370:3
          #5 0x84a10c in __wt_page_in_func /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/btree/bt_read.c:592:4
          #6 0x900827 in __wt_page_swap_func /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/include/btree.i:1512:8
          #7 0x8fc2e8 in __wt_row_search /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/btree/row_srch.c:446:14
          #8 0xaf06a1 in __cursor_row_search /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:377:2
          #9 0xaf6435 in __wt_btcur_remove /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:1015:3
          #10 0x97dffa in __curfile_remove /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/cursor/cur_file.c:398:2
          #11 0x52c2a3 in row_remove /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:1698:9
          #12 0x527196 in ops /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:808:9
          #13 0x7f6bc7067dc4 in start_thread /usr/src/debug/glibc-2.17-c758a686/nptl/pthread_create.c:308
      
      Thread T58 created by T0 here:
          #0 0x4373c1 in __interceptor_pthread_create /home/bostic/src/llvm40/projects/compiler-rt/lib/asan/asan_interceptors.cc:305
          #1 0x64d040 in __wt_thread_create /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/os_posix/os_thread.c:30:2
          #2 0x522a3b in wts_ops /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:164:3
          #3 0x5334dd in main /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/test/format/../../../test/format/t.c:209:5
          #4 0x7f6bc6178b34 in __libc_start_main /usr/src/debug/glibc-2.17-c758a686/csu/../csu/libc-start.c:274
      
      SUMMARY: AddressSanitizer: heap-buffer-overflow /mnt/data0/jenkins/workspace/wiredtiger-test-race-condition-stress-sanitizer/build_posix/../src/include/btree.i:1001:9 in __wt_row_leaf_key
      Shadow bytes around the buggy address:
        0x0c0e800234a0: fa fa 00 00 00 00 00 00 00 00 00 07 fa fa fa fa
        0x0c0e800234b0: 00 00 00 00 00 00 00 00 00 00 fa fa fa fa 00 00
        0x0c0e800234c0: 00 00 00 00 00 00 00 00 fa fa fa fa 00 00 00 00
        0x0c0e800234d0: 00 00 00 00 00 00 fa fa fa fa 00 00 00 00 00 00
        0x0c0e800234e0: 00 00 06 fa fa fa fa fa 00 00 00 00 00 00 00 00
      =>0x0c0e800234f0: 00 00[fa]fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x0c0e80023500: fa fa fa fa fd fd fd fd fd fd fd fd fd fd fa fa
        0x0c0e80023510: fa fa 00 00 00 00 00 00 00 00 00 00 fa fa fa fa
        0x0c0e80023520: fd fd fd fd fd fd fd fd fd fd fa fa fa fa 00 00
        0x0c0e80023530: 00 00 00 00 00 00 00 00 fa fa fa fa 00 00 00 00
        0x0c0e80023540: 00 00 00 00 00 00 fa fa fa fa 00 00 00 00 00 00
      Shadow byte legend (one shadow byte represents 8 application bytes):
        Addressable:           00
        Partially addressable: 01 02 03 04 05 06 07 
        Heap left redzone:       fa
        Freed heap region:       fd
        Stack left redzone:      f1
        Stack mid redzone:       f2
        Stack right redzone:     f3
        Stack after return:      f5
        Stack use after scope:   f8
        Global redzone:          f9
        Global init order:       f6
        Poisoned by user:        f7
        Container overflow:      fc
        Array cookie:            ac
        Intra object redzone:    bb
        ASan internal:           fe
        Left alloca redzone:     ca
        Right alloca redzone:    cb
      ==40111==ABORTING
      

            Assignee:
            keith.bostic@mongodb.com Keith Bostic (Inactive)
            Reporter:
            neha.khatri Neha Khatri
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Created:
              Updated:
              Resolved: