Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-71636

Explain executionStats does not work for $lookup on sharded collection

    • Fully Compatible
    • ALL
    • v6.3, v6.0
    • Hide

      Sample data:

      db.getCollection('tdube04mipmed0.332-0750.long').insertOne(
         {
            "_id": ObjectId("638468a8c6ac4d7abf8035d5"),
            "data": [
               {
                  "_id": ObjectId("6384689b60dc7b330284cab6"),
                  "t0": ISODate("2022-11-18T00:00:00.000+0100"),
                  "t": ISODate("2022-11-18T23:59:59.999+0100"),
                  "a": "periodical",
                  "di": "72.163.4.333",
                  "tsp": NumberInt(62160),
                  "tsi": "888.141.227.4",
                  "dp": NumberInt(443),
                  "f": "ALTS01TBMFW03",
                  "n": "snat",
                  "p": "tcp",
                  "si": "10.999.999.1",
                  "sp": NumberInt(42680),
                  "tt": NumberInt(10),
                  "h": NumberInt(102),
                  "ts": [ISODate("2022-11-14T08:31:00.000+0100"), ISODate("2022-11-28T08:21:00.000+0100")],
                  "si_subnet": "10.999.960.0/18"
               }
            ],
            "target": "sessions.20221118"
         }
      )
      
      db.getCollection('sessions.20221118').insertOne(
         {
            "_id": ObjectId("638468a8c6ac4d7abf803842"),
            "si": "10.999.999.1",
            "tsi": "888.141.227.4",
            "di": "72.163.4.333",
            "dp": NumberInt(443),
            "h": NumberInt(102),
            "n": "snat",
            "sp": NumberInt(42680),
            "t": ISODate("2022-11-18T23:59:59.999+0100"),
            "t0": ISODate("2022-11-18T00:00:00.000+0100"),
            "ts": [ISODate("2022-11-14T08:31:00.000+0100"), ISODate("2022-11-28T08:21:00.000+0100")],
            "tsp": NumberInt(62160)
         }
      )
      
      db.runCommand({ createIndexes: 'sessions.20221118', indexes: [{ name: "tsi_si", key: { tsi: 1, si: 1 } }] })
      
      sh.shardCollection(`${db.getName()}.sessions.20221118`, { tsi: 1, si: 1 })
       
      Show
      Sample data: db.getCollection( 'tdube04mipmed0.332-0750. long ' ).insertOne(    {       "_id" : ObjectId( "638468a8c6ac4d7abf8035d5" ),       "data" : [          {             "_id" : ObjectId( "6384689b60dc7b330284cab6" ),             "t0" : ISODate( "2022-11-18T00:00:00.000+0100" ),             "t" : ISODate( "2022-11-18T23:59:59.999+0100" ),             "a" : "periodical" ,             "di" : "72.163.4.333" ,             "tsp" : NumberInt(62160),             "tsi" : "888.141.227.4" ,             "dp" : NumberInt(443),             "f" : "ALTS01TBMFW03" ,             "n" : "snat" ,             "p" : "tcp" ,             "si" : "10.999.999.1" ,             "sp" : NumberInt(42680),             "tt" : NumberInt(10),             "h" : NumberInt(102),             "ts" : [ISODate( "2022-11-14T08:31:00.000+0100" ), ISODate( "2022-11-28T08:21:00.000+0100" )],             "si_subnet" : "10.999.960.0/18"          }       ],       "target" : "sessions.20221118"    } ) db.getCollection( 'sessions.20221118' ).insertOne(    {       "_id" : ObjectId( "638468a8c6ac4d7abf803842" ),       "si" : "10.999.999.1" ,       "tsi" : "888.141.227.4" ,       "di" : "72.163.4.333" ,       "dp" : NumberInt(443),       "h" : NumberInt(102),       "n" : "snat" ,       "sp" : NumberInt(42680),       "t" : ISODate( "2022-11-18T23:59:59.999+0100" ),       "t0" : ISODate( "2022-11-18T00:00:00.000+0100" ),       "ts" : [ISODate( "2022-11-14T08:31:00.000+0100" ), ISODate( "2022-11-28T08:21:00.000+0100" )],       "tsp" : NumberInt(62160)    } ) db.runCommand({ createIndexes: 'sessions.20221118' , indexes: [{ name: "tsi_si" , key: { tsi: 1, si: 1 } }] }) sh.shardCollection(`${db.getName()}.sessions.20221118`, { tsi: 1, si: 1 })
    • QE 2023-02-06, QE 2023-02-20, QE 2023-03-06, QE 2023-03-20, QE 2023-04-03

      I like to get detailed execution plan for this aggregation pipeline:

      db.getSiblingDB('data').getCollection('tdube04mipmed0.332-0750.long').aggregate([
         { "$match": { "_id": ObjectId("638468a8c6ac4d7abf8035d5") } },
         { "$unwind": "$data" },
         { "$replaceWith": "$data" },
         {
            "$lookup": {
               "from": "sessions.20221118",
               "let": { "si": "$si", "sp": "$sp", "di": "$di", "dp": "$dp", "n": "$n", "t0": "$t0", "t": "$t", "ts": "$ts", "tsp": "$tsp", "tsi": "$tsi" },
               "pipeline": [
                  {
                     "$match": {
                        "n": "snat",
                        "ts": { "$exists": true },
                        "h": { "$in": [102, 103] }
                     }
                  },
                  {
                     "$match": {
                        "$expr": {
                           "$and": [
                              { "$eq": ["$si", "$$si"] },
                              { "$eq": ["$sp", "$$sp"] },
                              { "$eq": ["$di", "$$di"] },
                              { "$eq": ["$dp", "$$dp"] },
                              { "$eq": ["$n", "$$n"] },
                              { "$gte": ["$t", "$$t"] },
                              { "$eq": ["$tsp", "$$tsp"] },
                              { "$eq": ["$tsi", "$$tsi"] }]
                        }
                     }
                  },
                  { "$sort": { "last": -1 } },
                  { "$limit": 1 }],
               "as": "longdata"
            }
         },
         { "$set": { "longdata": { "$first": "$longdata" } } },
         {
            "$set": {
               "_id": { "$ifNull": ["$longdata._id", "$$REMOVE"] },
               "longdata": "$$REMOVE",
               "ts": [
                  { "$min": [{ "$first": "$ts" }, { "$first": { "$ifNull": ["$longdata.ts", "$ts"] } }] },
                  { "$max": [{ "$last": "$ts" }, { "$last": { "$ifNull": ["$longdata.ts", "$ts"] } }] }
               ],
               "tp": { "$cond": ["$longdata._id", { "$concatArrays": [{ "$ifNull": ["$longdata.tp", []] }, [{ "$last": { "$ifNull": ["$longdata.ts", "$ts"] } }]] }, "$$REMOVE"] }
            }
         },
         { "$unset": ["tt", "si_subnet", "f", "a", "p", "rb", "sb"] },
         { "$merge": { "into": { "db": "data", "coll": "sessions.20221118" } } }
      ], { explain: 'executionStats' }) 

      But I get this error:

      MongoServerError: Use of undefined variable: si
          at Connection.onMessage (C:\Programs\MongoDB\Server\bin\mongosh.exe:72962:20)
          at MessageStream.<anonymous> (C:\Programs\MongoDB\Server\bin\mongosh.exe:72777:56)
          at MessageStream.emit (node:events:513:28)
          at MessageStream.emit (node:domain:552:15)
          at processIncomingData (C:\Programs\MongoDB\Server\bin\mongosh.exe:72586:14)
          at MessageStream._write (C:\Programs\MongoDB\Server\bin\mongosh.exe:72462:5)
          at writeOrBuffer (node:internal/streams/writable:391:12)
          at _write (node:internal/streams/writable:332:10)
          at MessageStream.Writable.write (node:internal/streams/writable:336:10)
          at Socket.ondata (node:internal/streams/readable:754:22)
       

      Same applies for {{

      {explain: 'allPlansExecution' }

      }}

      It works fine on unsharded collections, but when lookup table is sharded then I get this error.

        1. reproduce.js
          4 kB
          Yuan Fang

            Assignee:
            david.storch@mongodb.com David Storch
            Reporter:
            wernfried.domscheit@sunrise.net Wernfried Domscheit
            Votes:
            0 Vote for this issue
            Watchers:
            14 Start watching this issue

              Created:
              Updated:
              Resolved: