Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-21331

"Empty keys not permitted" invariant failure after WriteConflictException

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • 3.2.0-rc3
    • Affects Version/s: None
    • Component/s: Querying, Storage
    • None
    • Fully Compatible
    • ALL
    • Hide

      1 - Apply this patch to master (d6b9e37e):

      diff --git a/jstests/concurrency/fsm_all_replication.js b/jstests/concurrency/fsm_all_replication.js
      index e59fe0a..0bd1549 100644
      --- a/jstests/concurrency/fsm_all_replication.js
      +++ b/jstests/concurrency/fsm_all_replication.js
      @@ -11,5 +11,5 @@ var blacklist = [
       ].map(function(file) { return dir + '/' + file; });
       
       runWorkloadsSerially(ls(dir).filter(function(file) {
      -    return !Array.contains(blacklist, file);
      +    return !Array.contains(blacklist, file) && file.indexOf('update_grow') > -1;
       }), { replication: true });
      diff --git a/jstests/concurrency/fsm_workloads/findAndModify_update_grow.js b/jstests/concurrency/fsm_workloads/findAndModify_update_grow.js
      index b1b0c0a..4c6af91 100644
      --- a/jstests/concurrency/fsm_workloads/findAndModify_update_grow.js
      +++ b/jstests/concurrency/fsm_workloads/findAndModify_update_grow.js
      @@ -110,15 +110,24 @@ var $config = (function() {
       
           })();
       
      +    function setup(db, collName, cluster) {
      +        cluster.executeOnMongodNodes(function lowerYieldParams(db) {
      +            assertAlways.commandWorked(
      +                db.adminCommand({ configureFailPoint: 'WTWriteConflictException',
      +                                  mode: { activationProbability: 0.25 }}));
      +        });
      +    }
      +
           var transitions = {
               insert: { findAndModify: 1 },
               findAndModify: { findAndModify: 1 }
           };
       
           return {
      -        threadCount: 20,
      +        threadCount: 100,
               iterations: 20,
               data: data,
      +        setup: setup,
               states: states,
               startState: 'insert',
               transitions: transitions
      

      2 - `python buildscripts/resmoke.py --suites=concurrency_replication`

      Show
      1 - Apply this patch to master (d6b9e37e): diff --git a/jstests/concurrency/fsm_all_replication.js b/jstests/concurrency/fsm_all_replication.js index e59fe0a..0bd1549 100644 --- a/jstests/concurrency/fsm_all_replication.js +++ b/jstests/concurrency/fsm_all_replication.js @@ -11,5 +11,5 @@ var blacklist = [ ].map(function(file) { return dir + '/' + file; }); runWorkloadsSerially(ls(dir).filter(function(file) { - return !Array.contains(blacklist, file); + return !Array.contains(blacklist, file) && file.indexOf('update_grow') > -1; }), { replication: true }); diff --git a/jstests/concurrency/fsm_workloads/findAndModify_update_grow.js b/jstests/concurrency/fsm_workloads/findAndModify_update_grow.js index b1b0c0a..4c6af91 100644 --- a/jstests/concurrency/fsm_workloads/findAndModify_update_grow.js +++ b/jstests/concurrency/fsm_workloads/findAndModify_update_grow.js @@ -110,15 +110,24 @@ var $config = (function() { })(); + function setup(db, collName, cluster) { + cluster.executeOnMongodNodes(function lowerYieldParams(db) { + assertAlways.commandWorked( + db.adminCommand({ configureFailPoint: 'WTWriteConflictException', + mode: { activationProbability: 0.25 }})); + }); + } + var transitions = { insert: { findAndModify: 1 }, findAndModify: { findAndModify: 1 } }; return { - threadCount: 20, + threadCount: 100, iterations: 20, data: data, + setup: setup, states: states, startState: 'insert', transitions: transitions 2 - `python buildscripts/resmoke.py --suites=concurrency_replication`
    • QuInt C (11/23/15)

      Log snippet:

      d20010| 2015-11-06T13:13:18.951-0500 I REPL     [SnapshotThread] skipping storage snapshot pass due to write conflict
      d20010| 2015-11-06T13:13:18.998-0500 I REPL     [SnapshotThread] skipping storage snapshot pass due to write conflict
      d20010| 2015-11-06T13:13:19.031-0500 I REPL     [SnapshotThread] skipping storage snapshot pass due to write conflict
      d20010| 2015-11-06T13:13:19.034-0500 E STORAGE  [conn65] WiredTiger (22) [1446833599:34616][48176:0x117e81000], WT_CURSOR.set_key: Empty keys not permitted: Invalid argument
      d20010| 2015-11-06T13:13:19.034-0500 E STORAGE  [conn103] WiredTiger (22) [1446833599:34640][48176:0x1191f3000], WT_CURSOR.set_key: Empty keys not permitted: Invalid argument
      d20010| 2015-11-06T13:13:19.035-0500 E STORAGE  [conn30] WiredTiger (22) [1446833599:35016][48176:0x116c98000], WT_CURSOR.set_key: Empty keys not permitted: Invalid argument
      d20010| 2015-11-06T13:13:19.035-0500 E STORAGE  [conn30] WiredTiger (22) [1446833599:35071][48176:0x116c98000], file:index-2018-5019316785757103492.wt, WT_CURSOR.search_near: requires key be set: Invalid argument
      d20010| 2015-11-06T13:13:19.035-0500 I -        [conn30] Invariant failure: ret resulted in status BadValue 22: Invalid argument at src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp 769
      

      Mangled backtrace:

      d20010|  mongod(_ZN5mongo15printStackTraceERNSt3__113basic_ostreamIcNS0_11char_traitsIcEEEE+0x3A) [0x10eb4c6ea]
      d20010|  mongod(_ZN5mongo10logContextEPKc+0x16A) [0x10eafd79a]
      d20010|  mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0x1CA) [0x10eaec66a]
      d20010|  mongod(_ZN5mongo12_GLOBAL__N_125WiredTigerIndexCursorBase12seekWTCursorERKNS_9KeyStringE+0xFA) [0x10e90e45a]
      d20010|  mongod(_ZN5mongo12_GLOBAL__N_125WiredTigerIndexCursorBase7restoreEv+0x5E) [0x10e90e21e]
      d20010|  mongod(_ZN5mongo12_GLOBAL__N_127WiredTigerIndexUniqueCursor7restoreEv+0x13) [0x10e90edf3]
      d20010|  mongod(_ZN5mongo9PlanStage12restoreStateEv+0x28) [0x10e4adda8]
      d20010|  mongod(_ZN5mongo9PlanStage12restoreStateEv+0x28) [0x10e4adda8]
      d20010|  mongod(_ZN5mongo12PlanExecutor27restoreStateWithoutRetryingEv+0x22) [0x10e6cf902]
      d20010|  mongod(_ZN5mongo15PlanYieldPolicy5yieldEPNS_13RecordFetcherE+0xF1) [0x10e6d5001]
      d20010|  mongod(_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x339) [0x10e6cfe29]
      d20010|  mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x39) [0x10e6cfa59]
      d20010|  mongod(_ZN5mongo7FindCmd3runEPNS_16OperationContextERKNSt3__112basic_stringIcNS3_11char_traitsIcEENS3_9allocatorIcEEEERNS_7BSONObjEiRS9_RNS_14BSONObjBuilderE+0x915) [0x10e3db7b5]
      d20010|  mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x9B9) [0x10e465ca9]
      d20010|  mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0xE31) [0x10e464ea1]
      d20010|  mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x2A2) [0x10e3b9542]
      d20010|  mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0x17C1) [0x10e55f7c1]
      d20010|  mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE+0xEE) [0x10e24fbde]
      d20010|  mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x253) [0x10eb0a143]
      d20010|  mongod(_ZNSt3__114__thread_proxyINS_5tupleIJNS_6__bindIPFPvS3_EJPN5mongo12_GLOBAL__N_124MessagingPortWithHandlerEEEEEEEEES3_S3_+0x61) [0x10eb09931]
      d20010|  libsystem_pthread.dylib(_pthread_body+0x83) [0x7fff8373f05a]
      d20010|  libsystem_pthread.dylib(_pthread_body+0x0) [0x7fff8373efd7]
      d20010|  libsystem_pthread.dylib(thread_start+0xD) [0x7fff8373c3ed]
      d20010| -----  END BACKTRACE  -----
      

      Version: d6b9e37e

            Assignee:
            mathias@mongodb.com Mathias Stearn
            Reporter:
            kamran.khan Kamran K.
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated:
              Resolved: