Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-54681

Resharding recipient shards which are also donor may fail retryable writes with IncompleteTransactionHistory too early

    • Fully Compatible
    • ALL
    • Hide
      python buildscripts/resmoke.py run --suite=sharding resharding_inplace_retryable_writes.js
      
      resharding_inplace_retryable_writes.js
      (function() {
      "use strict";
      
      load("jstests/sharding/libs/resharding_test_fixture.js");
      
      const reshardingTest = new ReshardingTest({numDonors: 2, numRecipients: 2, reshardInPlace: true});
      reshardingTest.setup();
      
      const donorShardNames = reshardingTest.donorShardNames;
      const sourceCollection = reshardingTest.createShardedCollection({
          ns: "reshardingDb.coll",
          shardKeyPattern: {oldKey: 1},
          chunks: [
              {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]},
              {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]},
          ],
      });
      
      assert.commandWorked(sourceCollection.insert([
          {_id: "stays on shard0", oldKey: -10, newKey: -10, counter: 0},
          {_id: "moves to shard0", oldKey: 10, newKey: -10, counter: 0},
      ]));
      
      const mongos = sourceCollection.getMongo();
      const session = mongos.startSession({causalConsistency: false, retryWrites: false});
      const sessionCollection = session.getDatabase(sourceCollection.getDB().getName())
                                    .getCollection(sourceCollection.getName());
      
      function runRetryableWrite(phase, expectedErrorCode = ErrorCodes.OK) {
          const res = sessionCollection.runCommand("update", {
              updates: [
                  {q: {_id: "stays on shard0"}, u: {$inc: {counter: 1}}},
                  {q: {_id: "moves to shard0"}, u: {$inc: {counter: 1}}},
              ],
              txnNumber: NumberLong(1)
          });
      
          if (expectedErrorCode === ErrorCodes.OK) {
              assert.commandWorked(res);
          } else {
              assert.commandFailedWithCode(res, expectedErrorCode);
          }
      
          const docs = sourceCollection.find().toArray();
          assert.eq(2, docs.length, {docs});
      
          for (const doc of docs) {
              assert.eq(1,
                        doc.counter,
                        {message: `retryable write executed more than once ${phase}`, id: doc._id, docs});
          }
      }
      
      runRetryableWrite("before resharding");
      
      const recipientShardNames = reshardingTest.recipientShardNames;
      reshardingTest.withReshardingInBackground(  //
          {
              newShardKeyPattern: {newKey: 1},
              newChunks: [
                  {min: {newKey: MinKey}, max: {newKey: 0}, shard: recipientShardNames[0]},
                  {min: {newKey: 0}, max: {newKey: MaxKey}, shard: recipientShardNames[1]},
              ],
          },
          () => {
              runRetryableWrite("during resharding");
      
              assert.soon(() => {
                  const coordinatorDoc = mongos.getCollection("config.reshardingOperations").findOne({
                      nss: sourceCollection.getFullName()
                  });
      
                  return coordinatorDoc !== null && coordinatorDoc.fetchTimestamp !== undefined;
              });
      
              runRetryableWrite("during resharding after fetchTimestamp was chosen");
      
              assert.soon(() => {
                  const coordinatorDoc = mongos.getCollection("config.reshardingOperations").findOne({
                      nss: sourceCollection.getFullName()
                  });
      
                  return coordinatorDoc !== null && coordinatorDoc.state === "applying";
              });
      
              runRetryableWrite("during resharding after collection cloning had finished");
          });
      
      runRetryableWrite("after resharding", ErrorCodes.IncompleteTransactionHistory);
      
      reshardingTest.teardown();
      })();
      
      Show
      python buildscripts/resmoke.py run --suite=sharding resharding_inplace_retryable_writes.js resharding_inplace_retryable_writes.js ( function () { "use strict" ; load( "jstests/sharding/libs/resharding_test_fixture.js" ); const reshardingTest = new ReshardingTest({numDonors: 2, numRecipients: 2, reshardInPlace: true }); reshardingTest.setup(); const donorShardNames = reshardingTest.donorShardNames; const sourceCollection = reshardingTest.createShardedCollection({ ns: "reshardingDb.coll" , shardKeyPattern: {oldKey: 1}, chunks: [ {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]}, {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]}, ], }); assert.commandWorked(sourceCollection.insert([ {_id: "stays on shard0" , oldKey: -10, newKey: -10, counter: 0}, {_id: "moves to shard0" , oldKey: 10, newKey: -10, counter: 0}, ])); const mongos = sourceCollection.getMongo(); const session = mongos.startSession({causalConsistency: false , retryWrites: false }); const sessionCollection = session.getDatabase(sourceCollection.getDB().getName()) .getCollection(sourceCollection.getName()); function runRetryableWrite(phase, expectedErrorCode = ErrorCodes.OK) { const res = sessionCollection.runCommand( "update" , { updates: [ {q: {_id: "stays on shard0" }, u: {$inc: {counter: 1}}}, {q: {_id: "moves to shard0" }, u: {$inc: {counter: 1}}}, ], txnNumber: NumberLong(1) }); if (expectedErrorCode === ErrorCodes.OK) { assert.commandWorked(res); } else { assert.commandFailedWithCode(res, expectedErrorCode); } const docs = sourceCollection.find().toArray(); assert.eq(2, docs.length, {docs}); for ( const doc of docs) { assert.eq(1, doc.counter, {message: `retryable write executed more than once ${phase}` , id: doc._id, docs}); } } runRetryableWrite( "before resharding" ); const recipientShardNames = reshardingTest.recipientShardNames; reshardingTest.withReshardingInBackground( // { newShardKeyPattern: {newKey: 1}, newChunks: [ {min: {newKey: MinKey}, max: {newKey: 0}, shard: recipientShardNames[0]}, {min: {newKey: 0}, max: {newKey: MaxKey}, shard: recipientShardNames[1]}, ], }, () => { runRetryableWrite( "during resharding" ); assert.soon(() => { const coordinatorDoc = mongos.getCollection( "config.reshardingOperations" ).findOne({ nss: sourceCollection.getFullName() }); return coordinatorDoc !== null && coordinatorDoc.fetchTimestamp !== undefined; }); runRetryableWrite( "during resharding after fetchTimestamp was chosen" ); assert.soon(() => { const coordinatorDoc = mongos.getCollection( "config.reshardingOperations" ).findOne({ nss: sourceCollection.getFullName() }); return coordinatorDoc !== null && coordinatorDoc.state === "applying" ; }); runRetryableWrite( "during resharding after collection cloning had finished" ); }); runRetryableWrite( "after resharding" , ErrorCodes.IncompleteTransactionHistory); reshardingTest.teardown(); })();
    • Sharding 2021-03-08, Sharding 2021-03-22
    • 1

      The RecipientStateMachine skips constructing a ReshardingTxnCloner for itself when the recipient shard is also a donor shard. The intention was to allow retryable writes started before the resharding operation to remain retryable during the resharding operation. However, this approach is insufficient because a recipient shard may write the kIncompleteHistoryStmtId entry due to the config.transactions entry of another donor shard that had also participated in the same retryable write.

      void ReshardingRecipientService::RecipientStateMachine::_initTxnCloner(
          OperationContext* opCtx, const Timestamp& fetchTimestamp) {
          auto catalogCache = Grid::get(opCtx)->catalogCache();
          auto routingInfo = catalogCache->getShardedCollectionRoutingInfo(opCtx, _recipientDoc.getNss());
          std::set<ShardId> shardList;
      
          const auto myShardId = ShardingState::get(opCtx)->shardId();
          routingInfo.getAllShardIds(&shardList);
          shardList.erase(myShardId);
      
          for (const auto& shard : shardList) {
              _txnCloners.push_back(
                  std::make_unique<ReshardingTxnCloner>(ReshardingSourceId(_id, shard), fetchTimestamp));
          }
      }
      

      https://github.com/mongodb/mongo/blob/2b397f74e6a05b7c11e63a61d4aaf9443f58b150/src/mongo/db/s/resharding/resharding_recipient_service.cpp#L369

            Assignee:
            alex.taskov@mongodb.com Alexander Taskov (Inactive)
            Reporter:
            max.hirschhorn@mongodb.com Max Hirschhorn
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Created:
              Updated:
              Resolved: