From 331e3e6d667f70eb7be73e27bac70037742c1918 Mon Sep 17 00:00:00 2001 From: Jordi Serra Torrens Date: Tue, 22 Dec 2020 11:32:52 +0000 Subject: [PATCH] SERVER-53105: Read chunks with snapshot read concern in ConfigServerCatalogCache --- .../sharding/batch_write_command_sharded.js | 3 ++ jstests/sharding/mongos_validate_writes.js | 9 +++++ jstests/sharding/move_stale_mongos.js | 9 +++-- jstests/sharding/multi_mongos2.js | 5 ++- .../map_reduce_invalid_output_collection.js | 11 ++++++ jstests/sharding/split_stale_mongos.js | 7 ++-- ...restarted_shards_agree_on_shard_version.js | 2 +- .../stale_mongos_updates_and_removes.js | 3 ++ src/mongo/db/SConscript | 1 + src/mongo/db/repl/read_concern_args.cpp | 3 ++ src/mongo/db/repl/read_concern_args.h | 2 ++ src/mongo/db/rs_local_client.cpp | 25 +++++++++++-- src/mongo/db/rs_local_client.h | 2 +- .../s/config/config_server_test_fixture.cpp | 5 +-- .../resharding_donor_service_test.cpp | 13 +++---- src/mongo/db/s/shard_local.cpp | 4 +-- src/mongo/db/s/shard_local.h | 2 +- src/mongo/s/catalog/sharding_catalog_client.h | 23 ++++++------ .../catalog/sharding_catalog_client_impl.cpp | 13 +++---- .../s/catalog/sharding_catalog_client_impl.h | 6 ++-- .../catalog/sharding_catalog_client_mock.cpp | 6 ++-- .../s/catalog/sharding_catalog_client_mock.h | 6 ++-- src/mongo/s/catalog_cache.cpp | 16 +++++++++ src/mongo/s/client/shard.cpp | 4 +-- src/mongo/s/client/shard.h | 4 +-- src/mongo/s/client/shard_remote.cpp | 17 +++++++-- src/mongo/s/client/shard_remote.h | 15 ++++---- .../s/config_server_catalog_cache_loader.cpp | 36 +++++++++++++------ .../s/config_server_catalog_cache_loader.h | 14 ++++++++ 29 files changed, 195 insertions(+), 71 deletions(-) diff --git a/jstests/sharding/batch_write_command_sharded.js b/jstests/sharding/batch_write_command_sharded.js index 61cb0ac900..28ff28bfb1 100644 --- a/jstests/sharding/batch_write_command_sharded.js +++ b/jstests/sharding/batch_write_command_sharded.js @@ -150,6 +150,9 @@ for (var i = 0; i < oldChunks.length; i++) { // Ensure that the inserts have propagated to all secondary nodes st.configRS.awaitReplication(); +// Let the stale mongos learn about the current configTime, but don't refresh brokenColl +staleMongos.getCollection("another.coll").findOne(); + // Stale mongos can no longer bring itself up-to-date! // END SETUP diff --git a/jstests/sharding/mongos_validate_writes.js b/jstests/sharding/mongos_validate_writes.js index 0852dccf76..9e4208b2e4 100644 --- a/jstests/sharding/mongos_validate_writes.js +++ b/jstests/sharding/mongos_validate_writes.js @@ -43,6 +43,9 @@ coll.drop(); coll.createIndex({c: 1}); st.shardColl(coll, {c: 1}, {c: 0}, {c: 1}, coll.getDB(), true); +// Let staleMongosA know about the current configTime without refreshing staleCollA +staleMongosA.getCollection("another.coll").findOne(); + // Make sure we can successfully upsert, even though we have stale state assert.commandWorked(staleCollA.update({c: "c"}, {c: "c"}, true)); @@ -58,6 +61,9 @@ st.shardColl(coll, {d: 1}, {d: 0}, {d: 1}, coll.getDB(), true); // Make sure we can successfully update, even though we have stale state assert.commandWorked(coll.insert({d: "d"})); +// Let staleMongosA know about the current configTime without refreshing staleCollA +staleMongosA.getCollection("another.coll").findOne(); + assert.commandWorked(staleCollA.update({d: "d"}, {$set: {x: "x"}}, false, false)); assert.eq(staleCollA.findOne().x, "x"); @@ -76,6 +82,9 @@ st.shardColl(coll, {e: 1}, {e: 0}, {e: 1}, coll.getDB(), true); // Make sure we can successfully remove, even though we have stale state assert.commandWorked(coll.insert({e: "e"})); +// Let staleMongosA know about the current configTime without refreshing staleCollA +staleMongosA.getCollection("another.coll").findOne(); + assert.commandWorked(staleCollA.remove({e: "e"}, true)); assert.eq(null, staleCollA.findOne()); diff --git a/jstests/sharding/move_stale_mongos.js b/jstests/sharding/move_stale_mongos.js index ba8c920ddd..ed3a06efa1 100644 --- a/jstests/sharding/move_stale_mongos.js +++ b/jstests/sharding/move_stale_mongos.js @@ -14,13 +14,18 @@ var curShardIndex = 0; for (var i = 0; i < 100; i += 10) { assert.commandWorked(st.s0.getDB('admin').runCommand({split: testNs, middle: {_id: i}})); - st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the split + + // Ensure s1 knows about a configTime later than the previous split + st.s1.getCollection('another.coll').findOne(); + var nextShardIndex = (curShardIndex + 1) % 2; var toShard = (nextShardIndex == 0) ? st.shard0.name : st.shard1.name; assert.commandWorked(st.s1.getDB('admin').runCommand( {moveChunk: testNs, find: {_id: i + 5}, to: toShard, _waitForDelete: true})); curShardIndex = nextShardIndex; - st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the move + + // Ensure s0 knows about a configTime later than the previous moveChunk + st.s0.getCollection('another.coll').findOne(); } st.stop(); diff --git a/jstests/sharding/multi_mongos2.js b/jstests/sharding/multi_mongos2.js index 1c85a3dfd2..f21cd6f259 100644 --- a/jstests/sharding/multi_mongos2.js +++ b/jstests/sharding/multi_mongos2.js @@ -50,7 +50,10 @@ assert.commandWorked(st.s1.adminCommand({shardcollection: "test.existing3", key: st.configRS.awaitLastOpCommitted(); assert.commandWorked(st.s1.adminCommand({split: "test.existing3", middle: {_id: 5}})); -st.configRS.awaitLastOpCommitted(); + +// Ensure s0 has seen a configTime later than the split +st.s0.getCollection("another.coll").findOne(); + assert.commandWorked( st.s0.adminCommand({moveChunk: "test.existing3", find: {_id: 1}, to: st.shard0.shardName})); diff --git a/jstests/sharding/query/map_reduce_invalid_output_collection.js b/jstests/sharding/query/map_reduce_invalid_output_collection.js index 8e06403658..ce5521ac33 100644 --- a/jstests/sharding/query/map_reduce_invalid_output_collection.js +++ b/jstests/sharding/query/map_reduce_invalid_output_collection.js @@ -60,6 +60,9 @@ assert.commandFailedWithCode( {mapReduce: "coll", map: map, reduce: reduce, out: {reduce: outColl, sharded: true}}), 31313); +// Let staleMongos1 learn about the latest configTime without refreshing coll +staleMongos1.getCollection("another.coll").findOne(); + // Expect a similar failure through a stale mongos. assert.commandFailedWithCode( staleMongos1.getDB(dbName).runCommand( @@ -89,6 +92,10 @@ function testAgainstValidShardedOutput(shardKey) { // Run the same mapReduce through a stale mongos and expect it to pass as well. assert.commandWorked(st.s.getDB(dbName).getCollection(outColl).remove({})); + + // Let staleMongos1 learn about the latest configTime without refreshing 'coll' + staleMongos1.getCollection("another.coll").findOne(); + assert.commandWorked(staleMongos1.getDB(dbName).runCommand( {mapReduce: "coll", map: map, reduce: reduce, out: {merge: outColl, sharded: true}})); } @@ -119,6 +126,10 @@ testAgainstValidShardedOutput({_id: "hashed"}); // Run the same mapReduce through a stale mongos and expect it to fail as well. Make sure to // leave at least one document in the target collection for the same reason as above. assert.commandWorked(st.s.getDB(dbName).getCollection(outColl).remove({_id: {$gt: 0}})); + + // Let staleMongos1 learn about the latest configTime without refreshing 'coll' + staleMongos1.getCollection("another.coll").findOne(); + assert.commandFailedWithCode( staleMongos1.getDB(dbName).runCommand( {mapReduce: "coll", map: map, reduce: reduce, out: {merge: outColl, sharded: true}}), diff --git a/jstests/sharding/split_stale_mongos.js b/jstests/sharding/split_stale_mongos.js index ddfe820c05..34410df150 100644 --- a/jstests/sharding/split_stale_mongos.js +++ b/jstests/sharding/split_stale_mongos.js @@ -12,9 +12,12 @@ assert.commandWorked(admin.runCommand({shardCollection: testNs, key: {_id: 1}})) for (var i = 0; i < 100; i += 10) { assert.commandWorked(st.s0.getDB('admin').runCommand({split: testNs, middle: {_id: i}})); - st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the previous split + // Ensure s1 has seen a configTime later than the split + st.s1.getCollection('another.coll').findOne(); + assert.commandWorked(st.s1.getDB('admin').runCommand({split: testNs, middle: {_id: i + 5}})); - st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the previous split + // Ensure s0 other mongos has seen a configTime later than the split + st.s0.getCollection('another.coll').findOne(); } st.stop(); \ No newline at end of file diff --git a/jstests/sharding/stale_mongos_and_restarted_shards_agree_on_shard_version.js b/jstests/sharding/stale_mongos_and_restarted_shards_agree_on_shard_version.js index 4648cb1727..14cd3a6b71 100644 --- a/jstests/sharding/stale_mongos_and_restarted_shards_agree_on_shard_version.js +++ b/jstests/sharding/stale_mongos_and_restarted_shards_agree_on_shard_version.js @@ -215,7 +215,7 @@ const staleMongoS = st.s1; // Check for the expected number of refreshes. const catalogCacheStatistics = st.shard0.adminCommand({serverStatus: 1}).shardingStatistics.catalogCache; - assert.eq(kNumThreadsForConvoyTest, catalogCacheStatistics.countFullRefreshesStarted); + assert.lte(kNumThreadsForConvoyTest, catalogCacheStatistics.countFullRefreshesStarted); assert.eq(0, catalogCacheStatistics.countIncrementalRefreshesStarted); } diff --git a/jstests/sharding/stale_mongos_updates_and_removes.js b/jstests/sharding/stale_mongos_updates_and_removes.js index 06878fe917..ec8181e0b9 100644 --- a/jstests/sharding/stale_mongos_updates_and_removes.js +++ b/jstests/sharding/stale_mongos_updates_and_removes.js @@ -61,6 +61,7 @@ function makeStaleMongosTargetMultipleShardsWhenAllChunksAreOnOneShard() { st.configRS.awaitLastOpCommitted(); // Use freshMongos to consolidate the chunks on one shard. + freshMongos.getCollection(collNS).findOne(); // Ensure freshMongos is not stale assert.commandWorked(freshMongos.adminCommand( {moveChunk: collNS, find: {x: 0}, to: st.shard0.shardName, _waitForDelete: true})); } @@ -82,6 +83,7 @@ function makeStaleMongosTargetOneShardWhenAllChunksAreOnAnotherShard() { assert(chunk.shard === st.shard0.shardName); // Use freshMongos to move chunk to another shard. + freshMongos.getCollection(collNS).findOne(); // Ensure freshMongos is not stale assert.commandWorked(freshMongos.adminCommand( {moveChunk: collNS, find: {x: 0}, to: st.shard1.shardName, _waitForDelete: true})); } @@ -103,6 +105,7 @@ function makeStaleMongosTargetOneShardWhenChunksAreOnMultipleShards() { assert(chunk.shard === st.shard0.shardName); // Use freshMongos to split and move chunks to both shards. + freshMongos.getCollection(collNS).findOne(); // Ensure freshMongos is not stale assert.commandWorked(freshMongos.adminCommand({split: collNS, middle: {x: splitPoint}})); assert.commandWorked(freshMongos.adminCommand( {moveChunk: collNS, find: {x: 0}, to: st.shard1.shardName, _waitForDelete: true})); diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index d2c4e1bda4..dd6c7d3764 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -1641,6 +1641,7 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', + 'catalog_raii', 'dbdirectclient', ], ) diff --git a/src/mongo/db/repl/read_concern_args.cpp b/src/mongo/db/repl/read_concern_args.cpp index 27c949b1d6..19f975f8de 100644 --- a/src/mongo/db/repl/read_concern_args.cpp +++ b/src/mongo/db/repl/read_concern_args.cpp @@ -64,6 +64,9 @@ const BSONObj ReadConcernArgs::kImplicitDefault; ReadConcernArgs::ReadConcernArgs() : _specified(false) {} +ReadConcernArgs::ReadConcernArgs(ReadConcernLevel level) + : _level(std::move(level)), _specified(_level) {} + ReadConcernArgs::ReadConcernArgs(boost::optional level) : _level(std::move(level)), _specified(_level) {} diff --git a/src/mongo/db/repl/read_concern_args.h b/src/mongo/db/repl/read_concern_args.h index 5c8b0d101f..562f11d234 100644 --- a/src/mongo/db/repl/read_concern_args.h +++ b/src/mongo/db/repl/read_concern_args.h @@ -76,6 +76,8 @@ public: ReadConcernArgs(); + ReadConcernArgs(ReadConcernLevel level); + ReadConcernArgs(boost::optional level); ReadConcernArgs(boost::optional opTime, boost::optional level); diff --git a/src/mongo/db/rs_local_client.cpp b/src/mongo/db/rs_local_client.cpp index 41d5ed47cc..c73a3da086 100644 --- a/src/mongo/db/rs_local_client.cpp +++ b/src/mongo/db/rs_local_client.cpp @@ -31,6 +31,7 @@ #include "mongo/db/rs_local_client.h" +#include "mongo/db/catalog_raii.h" #include "mongo/db/curop.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/repl/repl_client_info.h" @@ -93,16 +94,30 @@ StatusWith RSLocalClient::runCommandOnce(OperationContex StatusWith RSLocalClient::queryOnce( OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcernLevel, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, boost::optional limit) { + const auto readConcernLevel = readConcernArgs.getLevel(); auto replCoord = repl::ReplicationCoordinator::get(opCtx); + auto readSourceScope = [&]() -> ReadSourceScope { + if (readConcernLevel == repl::ReadConcernLevel::kMajorityReadConcern) { + return ReadSourceScope(opCtx, RecoveryUnit::ReadSource::kMajorityCommitted); + } else if (readConcernLevel == repl::ReadConcernLevel::kSnapshotReadConcern) { + invariant(readConcernArgs.getArgsAtClusterTime()); + return ReadSourceScope(opCtx, + RecoveryUnit::ReadSource::kProvided, + readConcernArgs.getArgsAtClusterTime()->asTimestamp()); + } else { + invariant(readConcernLevel == repl::ReadConcernLevel::kLocalReadConcern); + return ReadSourceScope(opCtx, RecoveryUnit::ReadSource::kNoTimestamp); + } + }(); + if (readConcernLevel == repl::ReadConcernLevel::kMajorityReadConcern) { // Set up operation context with majority read snapshot so correct optime can be retrieved. - opCtx->recoveryUnit()->setTimestampReadSource(RecoveryUnit::ReadSource::kMajorityCommitted); Status status = opCtx->recoveryUnit()->majorityCommittedSnapshotAvailable(); // Wait for any writes performed by this ShardLocal instance to be committed and visible. @@ -118,6 +133,12 @@ StatusWith RSLocalClient::queryOnce( if (!status.isOK()) { return status; } + } else if (readConcernLevel == repl::ReadConcernLevel::kSnapshotReadConcern) { + invariant(readConcernArgs.getArgsAtClusterTime()); + Status readConcernStatus = replCoord->waitUntilOpTimeForRead(opCtx, readConcernArgs); + if (!readConcernStatus.isOK()) { + return readConcernStatus; + } } else { invariant(readConcernLevel == repl::ReadConcernLevel::kLocalReadConcern); } diff --git a/src/mongo/db/rs_local_client.h b/src/mongo/db/rs_local_client.h index 233732b5c8..a8d5fe7391 100644 --- a/src/mongo/db/rs_local_client.h +++ b/src/mongo/db/rs_local_client.h @@ -63,7 +63,7 @@ public: */ StatusWith queryOnce(OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcernLevel, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, diff --git a/src/mongo/db/s/config/config_server_test_fixture.cpp b/src/mongo/db/s/config/config_server_test_fixture.cpp index 4022eb3a25..a93b2f4489 100644 --- a/src/mongo/db/s/config/config_server_test_fixture.cpp +++ b/src/mongo/db/s/config/config_server_test_fixture.cpp @@ -139,8 +139,9 @@ void ConfigServerTestFixture::_setUp(std::function onPreInitGlobalStateF _addShardNetworkTestEnv = std::make_unique(_executorForAddShard, _mockNetworkForAddShard); - CatalogCacheLoader::set(getServiceContext(), - std::make_unique()); + auto configServerCatalogCacheLoader = std::make_unique(); + configServerCatalogCacheLoader->setAvoidSnapshotForRefresh_ForTest(); + CatalogCacheLoader::set(getServiceContext(), std::move(configServerCatalogCacheLoader)); onPreInitGlobalStateFn(); diff --git a/src/mongo/db/s/resharding/resharding_donor_service_test.cpp b/src/mongo/db/s/resharding/resharding_donor_service_test.cpp index 8fb8f0c69a..3d5ca76137 100644 --- a/src/mongo/db/s/resharding/resharding_donor_service_test.cpp +++ b/src/mongo/db/s/resharding/resharding_donor_service_test.cpp @@ -90,12 +90,13 @@ protected: return chunks; } - StatusWith> getChunks(OperationContext* opCtx, - const BSONObj& filter, - const BSONObj& sort, - boost::optional limit, - repl::OpTime* opTime, - repl::ReadConcernLevel readConcern) override { + StatusWith> getChunks( + OperationContext* opCtx, + const BSONObj& filter, + const BSONObj& sort, + boost::optional limit, + repl::OpTime* opTime, + const repl::ReadConcernArgs& readConcern) override { auto version = ChunkVersion(1, 0, OID::gen(), boost::none /* timestamp */); return makeChunks(reshardingTempNss(_existingUUID), _recipients, version); } diff --git a/src/mongo/db/s/shard_local.cpp b/src/mongo/db/s/shard_local.cpp index 1467b1cdd6..2a8aab319a 100644 --- a/src/mongo/db/s/shard_local.cpp +++ b/src/mongo/db/s/shard_local.cpp @@ -125,12 +125,12 @@ StatusWith ShardLocal::_runExhaustiveCursorCommand( StatusWith ShardLocal::_exhaustiveFindOnConfig( OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcernLevel, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, boost::optional limit) { - return _rsLocalClient.queryOnce(opCtx, readPref, readConcernLevel, nss, query, sort, limit); + return _rsLocalClient.queryOnce(opCtx, readPref, readConcernArgs, nss, query, sort, limit); } Status ShardLocal::createIndexOnConfig(OperationContext* opCtx, diff --git a/src/mongo/db/s/shard_local.h b/src/mongo/db/s/shard_local.h index ef50595971..f451e10460 100644 --- a/src/mongo/db/s/shard_local.h +++ b/src/mongo/db/s/shard_local.h @@ -91,7 +91,7 @@ private: StatusWith _exhaustiveFindOnConfig( OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcernLevel, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, diff --git a/src/mongo/s/catalog/sharding_catalog_client.h b/src/mongo/s/catalog/sharding_catalog_client.h index 43c2c160a1..a844f27f6a 100644 --- a/src/mongo/s/catalog/sharding_catalog_client.h +++ b/src/mongo/s/catalog/sharding_catalog_client.h @@ -128,10 +128,10 @@ public: * the failure. These are some of the known failures: * - NamespaceNotFound - collection does not exist */ - virtual CollectionType getCollection( - OperationContext* opCtx, - const NamespaceString& nss, - repl::ReadConcernLevel readConcernLevel = repl::ReadConcernLevel::kMajorityReadConcern) = 0; + virtual CollectionType getCollection(OperationContext* opCtx, + const NamespaceString& nss, + const repl::ReadConcernArgs& readConcern = + repl::ReadConcernLevel::kMajorityReadConcern) = 0; /** * Retrieves all collections under a specified database (or in the system). If the dbName @@ -173,12 +173,13 @@ public: * * Returns a vector of ChunkTypes, or a !OK status if an error occurs. */ - virtual StatusWith> getChunks(OperationContext* opCtx, - const BSONObj& filter, - const BSONObj& sort, - boost::optional limit, - repl::OpTime* opTime, - repl::ReadConcernLevel readConcern) = 0; + virtual StatusWith> getChunks( + OperationContext* opCtx, + const BSONObj& filter, + const BSONObj& sort, + boost::optional limit, + repl::OpTime* opTime, + const repl::ReadConcernArgs& readConcern) = 0; /** * Retrieves all zones defined for the specified collection. The returned vector is sorted based @@ -337,7 +338,7 @@ private: virtual StatusWith>> _exhaustiveFindOnConfig( OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcern, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, diff --git a/src/mongo/s/catalog/sharding_catalog_client_impl.cpp b/src/mongo/s/catalog/sharding_catalog_client_impl.cpp index ff7053ab8e..2cd11976c6 100644 --- a/src/mongo/s/catalog/sharding_catalog_client_impl.cpp +++ b/src/mongo/s/catalog/sharding_catalog_client_impl.cpp @@ -243,11 +243,11 @@ StatusWith> ShardingCatalogClientImpl::_fetchData CollectionType ShardingCatalogClientImpl::getCollection(OperationContext* opCtx, const NamespaceString& nss, - repl::ReadConcernLevel readConcernLevel) { + const repl::ReadConcernArgs& readConcern) { auto collDoc = uassertStatusOK(_exhaustiveFindOnConfig(opCtx, kConfigReadSelector, - readConcernLevel, + readConcern, CollectionType::ConfigNS, BSON(CollectionType::kNssFieldName << nss.ns()), BSONObj(), @@ -404,9 +404,10 @@ StatusWith> ShardingCatalogClientImpl::getChunks( const BSONObj& sort, boost::optional limit, OpTime* opTime, - repl::ReadConcernLevel readConcern) { + const repl::ReadConcernArgs& readConcern) { invariant(serverGlobalParams.clusterRole == ClusterRole::ConfigServer || - readConcern == repl::ReadConcernLevel::kMajorityReadConcern); + readConcern.getLevel() == repl::ReadConcernLevel::kMajorityReadConcern || + readConcern.getLevel() == repl::ReadConcernLevel::kSnapshotReadConcern); // Convert boost::optional to boost::optional. auto longLimit = limit ? boost::optional(*limit) : boost::none; @@ -873,13 +874,13 @@ Status ShardingCatalogClientImpl::removeConfigDocuments(OperationContext* opCtx, StatusWith>> ShardingCatalogClientImpl::_exhaustiveFindOnConfig( OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcern, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, boost::optional limit) { auto response = Grid::get(opCtx)->shardRegistry()->getConfigShard()->exhaustiveFindOnConfig( - opCtx, readPref, readConcern, nss, query, sort, limit); + opCtx, readPref, readConcernArgs, nss, query, sort, limit); if (!response.isOK()) { return response.getStatus(); } diff --git a/src/mongo/s/catalog/sharding_catalog_client_impl.h b/src/mongo/s/catalog/sharding_catalog_client_impl.h index 66f25e71fb..dac706119c 100644 --- a/src/mongo/s/catalog/sharding_catalog_client_impl.h +++ b/src/mongo/s/catalog/sharding_catalog_client_impl.h @@ -73,7 +73,7 @@ public: CollectionType getCollection(OperationContext* opCtx, const NamespaceString& nss, - repl::ReadConcernLevel readConcernLevel) override; + const repl::ReadConcernArgs& readConcern) override; std::vector getCollections(OperationContext* opCtx, StringData db, @@ -90,7 +90,7 @@ public: const BSONObj& sort, boost::optional limit, repl::OpTime* opTime, - repl::ReadConcernLevel readConcern) override; + const repl::ReadConcernArgs& readConcern) override; StatusWith> getTagsForCollection(OperationContext* opCtx, const NamespaceString& nss) override; @@ -174,7 +174,7 @@ private: StatusWith>> _exhaustiveFindOnConfig( OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcern, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, diff --git a/src/mongo/s/catalog/sharding_catalog_client_mock.cpp b/src/mongo/s/catalog/sharding_catalog_client_mock.cpp index 7f3c5a9029..e7217b30c2 100644 --- a/src/mongo/s/catalog/sharding_catalog_client_mock.cpp +++ b/src/mongo/s/catalog/sharding_catalog_client_mock.cpp @@ -57,7 +57,7 @@ std::vector ShardingCatalogClientMock::getAllDBs(OperationContext* CollectionType ShardingCatalogClientMock::getCollection(OperationContext* opCtx, const NamespaceString& nss, - repl::ReadConcernLevel readConcernLevel) { + const repl::ReadConcernArgs& readConcern) { uasserted(ErrorCodes::InternalError, "Method not implemented"); } @@ -82,7 +82,7 @@ StatusWith> ShardingCatalogClientMock::getChunks( const BSONObj& sort, boost::optional limit, repl::OpTime* opTime, - repl::ReadConcernLevel readConcern) { + const repl::ReadConcernArgs& readConcern) { return {ErrorCodes::InternalError, "Method not implemented"}; } @@ -170,7 +170,7 @@ StatusWith> ShardingCatalogClientMock::getNe StatusWith>> ShardingCatalogClientMock::_exhaustiveFindOnConfig(OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcern, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, diff --git a/src/mongo/s/catalog/sharding_catalog_client_mock.h b/src/mongo/s/catalog/sharding_catalog_client_mock.h index ba7ec3ba81..badd006730 100644 --- a/src/mongo/s/catalog/sharding_catalog_client_mock.h +++ b/src/mongo/s/catalog/sharding_catalog_client_mock.h @@ -50,7 +50,7 @@ public: CollectionType getCollection(OperationContext* opCtx, const NamespaceString& nss, - repl::ReadConcernLevel readConcernLevel) override; + const repl::ReadConcernArgs& readConcernArgs) override; std::vector getCollections(OperationContext* opCtx, StringData db, @@ -67,7 +67,7 @@ public: const BSONObj& sort, boost::optional limit, repl::OpTime* opTime, - repl::ReadConcernLevel readConcern) override; + const repl::ReadConcernArgs& readConcern) override; StatusWith> getTagsForCollection(OperationContext* opCtx, const NamespaceString& nss) override; @@ -135,7 +135,7 @@ private: StatusWith>> _exhaustiveFindOnConfig( OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcern, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, diff --git a/src/mongo/s/catalog_cache.cpp b/src/mongo/s/catalog_cache.cpp index 2f9c7aca27..0abf7b14eb 100644 --- a/src/mongo/s/catalog_cache.cpp +++ b/src/mongo/s/catalog_cache.cpp @@ -207,6 +207,22 @@ StatusWith CatalogCache::_getCollectionRoutingInfoAt( std::move(collEntry), atClusterTime); } catch (ExceptionFor& ex) { + LOGV2_FOR_CATALOG_REFRESH( + 5310501, + 0, + "Collection refresh failed: ConflictingOperationInProgress", + "namespace"_attr = nss); + _stats.totalRefreshWaitTimeMicros.addAndFetch(t.micros()); + acquireTries++; + if (acquireTries == kMaxInconsistentRoutingInfoRefreshAttempts) { + return ex.toStatus(); + } + } catch (ExceptionForCat& ex) { + LOGV2_FOR_CATALOG_REFRESH(5310502, + 0, + "Collection refresh failed", + "namespace"_attr = nss, + "exception"_attr = redact(ex)); _stats.totalRefreshWaitTimeMicros.addAndFetch(t.micros()); acquireTries++; if (acquireTries == kMaxInconsistentRoutingInfoRefreshAttempts) { diff --git a/src/mongo/s/client/shard.cpp b/src/mongo/s/client/shard.cpp index 90159e313f..9e83c4bc05 100644 --- a/src/mongo/s/client/shard.cpp +++ b/src/mongo/s/client/shard.cpp @@ -233,7 +233,7 @@ BatchedCommandResponse Shard::runBatchWriteCommand(OperationContext* opCtx, StatusWith Shard::exhaustiveFindOnConfig( OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcernLevel, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, @@ -243,7 +243,7 @@ StatusWith Shard::exhaustiveFindOnConfig( for (int retry = 1; retry <= kOnErrorNumRetries; retry++) { auto result = - _exhaustiveFindOnConfig(opCtx, readPref, readConcernLevel, nss, query, sort, limit); + _exhaustiveFindOnConfig(opCtx, readPref, readConcernArgs, nss, query, sort, limit); if (retry < kOnErrorNumRetries && isRetriableError(result.getStatus().code(), RetryPolicy::kIdempotent)) { diff --git a/src/mongo/s/client/shard.h b/src/mongo/s/client/shard.h index d4ea6cedfc..eac69f4263 100644 --- a/src/mongo/s/client/shard.h +++ b/src/mongo/s/client/shard.h @@ -240,7 +240,7 @@ public: */ StatusWith exhaustiveFindOnConfig(OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcernLevel, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, @@ -314,7 +314,7 @@ private: virtual StatusWith _exhaustiveFindOnConfig( OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcernLevel, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, diff --git a/src/mongo/s/client/shard_remote.cpp b/src/mongo/s/client/shard_remote.cpp index 6487164a27..c495f8ed1a 100644 --- a/src/mongo/s/client/shard_remote.cpp +++ b/src/mongo/s/client/shard_remote.cpp @@ -349,7 +349,7 @@ StatusWith ShardRemote::_runExhaustiveCursorCommand( StatusWith ShardRemote::_exhaustiveFindOnConfig( OperationContext* opCtx, const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcernLevel, + const repl::ReadConcernArgs& readConcernArgs, const NamespaceString& nss, const BSONObj& query, const BSONObj& sort, @@ -357,10 +357,21 @@ StatusWith ShardRemote::_exhaustiveFindOnConfig( invariant(isConfig()); auto const grid = Grid::get(opCtx); + const auto readConcernLevel = readConcernArgs.getLevel(); BSONObj readConcernObj; { - invariant(readConcernLevel == repl::ReadConcernLevel::kMajorityReadConcern); - const auto readConcern = grid->readConcernWithConfigTime(readConcernLevel); + invariant(readConcernLevel == repl::ReadConcernLevel::kMajorityReadConcern || + readConcernLevel == repl::ReadConcernLevel::kSnapshotReadConcern); + const auto readConcern = [&]() { + if (readConcernLevel == repl::ReadConcernLevel::kMajorityReadConcern) { + return grid->readConcernWithConfigTime(readConcernLevel); + } else if (readConcernLevel == repl::ReadConcernLevel::kSnapshotReadConcern) { + return readConcernArgs; + } else { + MONGO_UNREACHABLE; + } + }(); + BSONObjBuilder bob; readConcern.appendInfo(&bob); readConcernObj = diff --git a/src/mongo/s/client/shard_remote.h b/src/mongo/s/client/shard_remote.h index da84f2b633..8af4df218b 100644 --- a/src/mongo/s/client/shard_remote.h +++ b/src/mongo/s/client/shard_remote.h @@ -115,14 +115,13 @@ private: Milliseconds maxTimeMSOverride, const BSONObj& cmdObj) final; - StatusWith _exhaustiveFindOnConfig( - OperationContext* opCtx, - const ReadPreferenceSetting& readPref, - const repl::ReadConcernLevel& readConcernLevel, - const NamespaceString& nss, - const BSONObj& query, - const BSONObj& sort, - boost::optional limit) final; + StatusWith _exhaustiveFindOnConfig(OperationContext* opCtx, + const ReadPreferenceSetting& readPref, + const repl::ReadConcernArgs& readConcernArgs, + const NamespaceString& nss, + const BSONObj& query, + const BSONObj& sort, + boost::optional limit) final; StatusWith _scheduleCommand( OperationContext* opCtx, diff --git a/src/mongo/s/config_server_catalog_cache_loader.cpp b/src/mongo/s/config_server_catalog_cache_loader.cpp index cd7133c2ff..715e35636d 100644 --- a/src/mongo/s/config_server_catalog_cache_loader.cpp +++ b/src/mongo/s/config_server_catalog_cache_loader.cpp @@ -37,6 +37,7 @@ #include "mongo/db/client.h" #include "mongo/db/operation_context.h" +#include "mongo/db/repl/replication_coordinator.h" #include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/grid.h" #include "mongo/util/fail_point.h" @@ -83,11 +84,26 @@ QueryAndSort createConfigDiffQueryUuid(const UUID& uuid, ChunkVersion collection */ CollectionAndChangedChunks getChangedChunks(OperationContext* opCtx, const NamespaceString& nss, - ChunkVersion sinceVersion) { + ChunkVersion sinceVersion, + bool avoidSnapshotForRefresh) { const auto catalogClient = Grid::get(opCtx)->catalogClient(); + const auto readConcernArgs = [&]() -> repl::ReadConcernArgs { + if (avoidSnapshotForRefresh) { + return {repl::ReadConcernLevel::kMajorityReadConcern}; + } else { + auto clusterTimeToRead = (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) + ? repl::ReplicationCoordinator::get(opCtx)->getMyLastAppliedOpTime() + : Grid::get(opCtx)->configOpTime(); + auto readConcernArgs = + repl::ReadConcernArgs(repl::ReadConcernLevel::kSnapshotReadConcern); + readConcernArgs.setArgsAtClusterTimeForSnapshot(clusterTimeToRead.getTimestamp()); + return readConcernArgs; + } + }(); + // Decide whether to do a full or partial load based on the state of the collection - const auto coll = catalogClient->getCollection(opCtx, nss); + const auto coll = catalogClient->getCollection(opCtx, nss, readConcernArgs); uassert(ErrorCodes::NamespaceNotFound, str::stream() << "Collection " << nss.ns() << " is dropped.", !coll.getDropped()); @@ -108,13 +124,9 @@ CollectionAndChangedChunks getChangedChunks(OperationContext* opCtx, // Query the chunks which have changed repl::OpTime opTime; - const std::vector changedChunks = uassertStatusOK( - Grid::get(opCtx)->catalogClient()->getChunks(opCtx, - diffQuery.query, - diffQuery.sort, - boost::none, - &opTime, - repl::ReadConcernLevel::kMajorityReadConcern)); + const std::vector changedChunks = + uassertStatusOK(Grid::get(opCtx)->catalogClient()->getChunks( + opCtx, diffQuery.query, diffQuery.sort, boost::none, &opTime, readConcernArgs)); uassert(ErrorCodes::ConflictingOperationInProgress, "No chunks were found for the collection", @@ -184,7 +196,7 @@ SemiFuture ConfigServerCatalogCacheLoader::getChunks getGlobalServiceContext()); auto opCtx = tc->makeOperationContext(); - return getChangedChunks(opCtx.get(), nss, version); + return getChangedChunks(opCtx.get(), nss, version, _avoidSnapshotForRefresh); }) .semi(); } @@ -202,4 +214,8 @@ SemiFuture ConfigServerCatalogCacheLoader::getDatabase(StringData .semi(); } +void ConfigServerCatalogCacheLoader::setAvoidSnapshotForRefresh_ForTest() { + _avoidSnapshotForRefresh = true; +} + } // namespace mongo diff --git a/src/mongo/s/config_server_catalog_cache_loader.h b/src/mongo/s/config_server_catalog_cache_loader.h index d16ae428b7..8c1384946f 100644 --- a/src/mongo/s/config_server_catalog_cache_loader.h +++ b/src/mongo/s/config_server_catalog_cache_loader.h @@ -55,9 +55,23 @@ public: ChunkVersion version) override; SemiFuture getDatabase(StringData dbName) override; + /** + * Don't use outside of unit_tests. + * TODO SERVER-54394 Remove this + */ + void setAvoidSnapshotForRefresh_ForTest(); + private: // Thread pool to be used to perform metadata load std::shared_ptr _executor; + + /* + * If 'true' avoids using snapshot read concern when refreshing the cache. Only to be used by + * unit_tests that use the ephemeralForTesting storage engine, because currently it doesn't + * support snapshot read concern. + * TODO SERVER-54394 Remove this. + */ + bool _avoidSnapshotForRefresh = false; }; } // namespace mongo -- 2.17.1