Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-64765

While running repair command after server crash - we are facing this issue

XMLWordPrintableJSON

    • Type: Icon: Question Question
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 4.4.13
    • Component/s: None
    • Environment:
      Ubuntu - AWS hosting

      Hello, 
      I'm trying to run Repair command after server crash - but getting WT_NOTFOUND error, I do not have any recent backup and 11 GB data is there. 

      Kindly let me know best way to restore the database. 

       

      {
  "t": {
    "$date": "2022-03-20T13:45:26.959-06:00"
  },
  "s": "I",
  "c": "NETWORK",
  "id": 4915701,
  "ctx": "main",
  "msg": "Initialized wire specification",
  "attr": {
    "spec": {
      "incomingExternalClient": {
        "minWireVersion": 0,
        "maxWireVersion": 13
      },
      "incomingInternalClient": {
        "minWireVersion": 0,
        "maxWireVersion": 13
      },
      "outgoing": {
        "minWireVersion": 0,
        "maxWireVersion": 13
      },
      "isInternalClient": true
    }
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:26.964-06:00"
  },
  "s": "I",
  "c": "CONTROL",
  "id": 23285,
  "ctx": "main",
  "msg": "Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'"
}{
  "t": {
    "$date": "2022-03-20T13:45:26.972-06:00"
  },
  "s": "W",
  "c": "ASIO",
  "id": 22601,
  "ctx": "main",
  "msg": "No TransportLayer configured during NetworkInterface startup"
}{
  "t": {
    "$date": "2022-03-20T13:45:26.973-06:00"
  },
  "s": "I",
  "c": "NETWORK",
  "id": 4648602,
  "ctx": "main",
  "msg": "Implicit TCP FastOpen in use."
}{
  "t": {
    "$date": "2022-03-20T13:45:26.976-06:00"
  },
  "s": "W",
  "c": "ASIO",
  "id": 22601,
  "ctx": "main",
  "msg": "No TransportLayer configured during NetworkInterface startup"
}{
  "t": {
    "$date": "2022-03-20T13:45:26.976-06:00"
  },
  "s": "I",
  "c": "REPL",
  "id": 5123008,
  "ctx": "main",
  "msg": "Successfully registered PrimaryOnlyService",
  "attr": {
    "service": "TenantMigrationDonorService",
    "ns": "config.tenantMigrationDonors"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:26.977-06:00"
  },
  "s": "I",
  "c": "REPL",
  "id": 5123008,
  "ctx": "main",
  "msg": "Successfully registered PrimaryOnlyService",
  "attr": {
    "service": "TenantMigrationRecipientService",
    "ns": "config.tenantMigrationRecipients"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:26.978-06:00"
  },
  "s": "I",
  "c": "CONTROL",
  "id": 5945603,
  "ctx": "main",
  "msg": "Multi threading initialized"
}{
  "t": {
    "$date": "2022-03-20T13:45:26.982-06:00"
  },
  "s": "I",
  "c": "CONTROL",
  "id": 4615611,
  "ctx": "initandlisten",
  "msg": "MongoDB starting",
  "attr": {
    "pid": 14924,
    "port": 27017,
    "dbPath": "C:/Users/ADMIN/backuphikardb/mongodb",
    "architecture": "64-bit",
    "host": "DESKTOP-48VEHET"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:26.982-06:00"
  },
  "s": "I",
  "c": "CONTROL",
  "id": 23398,
  "ctx": "initandlisten",
  "msg": "Target operating system minimum version",
  "attr": {
    "targetMinOS": "Windows 7/Windows Server 2008 R2"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:26.988-06:00"
  },
  "s": "I",
  "c": "CONTROL",
  "id": 23403,
  "ctx": "initandlisten",
  "msg": "Build Info",
  "attr": {
    "buildInfo": {
      "version": "5.0.6",
      "gitVersion": "212a8dbb47f07427dae194a9c75baec1d81d9259",
      "modules": [
        
      ],
      "allocator": "tcmalloc",
      "environment": {
        "distmod": "windows",
        "distarch": "x86_64",
        "target_arch": "x86_64"
      }
    }
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:26.990-06:00"
  },
  "s": "I",
  "c": "CONTROL",
  "id": 51765,
  "ctx": "initandlisten",
  "msg": "Operating System",
  "attr": {
    "os": {
      "name": "Microsoft Windows 10",
      "version": "10.0 (build 19044)"
    }
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:26.991-06:00"
  },
  "s": "I",
  "c": "CONTROL",
  "id": 21951,
  "ctx": "initandlisten",
  "msg": "Options set by command line",
  "attr": {
    "options": {
      "repair": true,
      "storage": {
        "dbPath": "C:\\Users\\ADMIN\\backuphikardb\\mongodb"
      }
    }
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.006-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22270,
  "ctx": "initandlisten",
  "msg": "Storage engine to use detected by data files",
  "attr": {
    "dbpath": "C:/Users/ADMIN/backuphikardb/mongodb",
    "storageEngine": "wiredTiger"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.007-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22315,
  "ctx": "initandlisten",
  "msg": "Opening WiredTiger",
  "attr": {
    "config": "create,cache_size=3529M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.052-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22430,
  "ctx": "initandlisten",
  "msg": "WiredTiger message",
  "attr": {
    "message": "[1647805527:52459][14924:140715253912912], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 456 through 457"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.412-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22430,
  "ctx": "initandlisten",
  "msg": "WiredTiger message",
  "attr": {
    "message": "[1647805527:412789][14924:140715253912912], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 457 through 457"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.518-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22430,
  "ctx": "initandlisten",
  "msg": "WiredTiger message",
  "attr": {
    "message": "[1647805527:517771][14924:140715253912912], txn-recover: [WT_VERB_RECOVERY_ALL] Main recovery loop: starting at 456/256 to 457/256"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.687-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22430,
  "ctx": "initandlisten",
  "msg": "WiredTiger message",
  "attr": {
    "message": "[1647805527:686773][14924:140715253912912], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 456 through 457"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.800-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22430,
  "ctx": "initandlisten",
  "msg": "WiredTiger message",
  "attr": {
    "message": "[1647805527:799776][14924:140715253912912], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 457 through 457"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.885-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22430,
  "ctx": "initandlisten",
  "msg": "WiredTiger message",
  "attr": {
    "message": "[1647805527:884832][14924:140715253912912], txn-recover: [WT_VERB_RECOVERY_ALL] Set global recovery timestamp: (0, 0)"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.885-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22430,
  "ctx": "initandlisten",
  "msg": "WiredTiger message",
  "attr": {
    "message": "[1647805527:885773][14924:140715253912912], txn-recover: [WT_VERB_RECOVERY_ALL] Set global oldest timestamp: (0, 0)"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.901-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22430,
  "ctx": "initandlisten",
  "msg": "WiredTiger message",
  "attr": {
    "message": "[1647805527:900786][14924:140715253912912], WT_SESSION.checkpoint: [WT_VERB_CHECKPOINT_PROGRESS] saving checkpoint snapshot min: 1, snapshot max: 1 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 39644794"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.916-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 4795906,
  "ctx": "initandlisten",
  "msg": "WiredTiger opened",
  "attr": {
    "durationMillis": 908
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.917-06:00"
  },
  "s": "I",
  "c": "RECOVERY",
  "id": 23987,
  "ctx": "initandlisten",
  "msg": "WiredTiger recoveryTimestamp",
  "attr": {
    "recoveryTimestamp": {
      "$timestamp": {
        "t": 0,
        "i": 0
      }
    }
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.920-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22316,
  "ctx": "initandlisten",
  "msg": "Repairing size cache"
}{
  "t": {
    "$date": "2022-03-20T13:45:27.930-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22327,
  "ctx": "initandlisten",
  "msg": "Verify succeeded. Not salvaging.",
  "attr": {
    "uri": "table:sizeStorer"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.934-06:00"
  },
  "s": "I",
  "c": "STORAGE",
  "id": 22246,
  "ctx": "initandlisten",
  "msg": "Repairing catalog metadata"
}{
  "t": {
    "$date": "2022-03-20T13:45:27.934-06:00"
  },
  "s": "W",
  "c": "STORAGE",
  "id": 22350,
  "ctx": "initandlisten",
  "msg": "Data file is missing. Attempting to drop and re-create the collection.",
  "attr": {
    "uri": "table:_mdb_catalog"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.942-06:00"
  },
  "s": "W",
  "c": "STORAGE",
  "id": 22353,
  "ctx": "initandlisten",
  "msg": "Rebuilding ident",
  "attr": {
    "ident": "_mdb_catalog"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.943-06:00"
  },
  "s": "E",
  "c": "STORAGE",
  "id": 22435,
  "ctx": "initandlisten",
  "msg": "WiredTiger error",
  "attr": {
    "error": -31803,
    "message": "[1647805527:943075][14924:140715253912912], file:WiredTiger.wt, WT_CURSOR.search: int __cdecl __schema_source_config(struct __wt_session_impl *,struct __wt_cursor *,const char *,const char **), 50: metadata information for source configuration \"file:_mdb_catalog.wt\" not found: WT_NOTFOUND: item not found"
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.943-06:00"
  },
  "s": "E",
  "c": "STORAGE",
  "id": 22357,
  "ctx": "initandlisten",
  "msg": "Rebuilding ident failed: failed to get metadata",
  "attr": {
    "uri": "table:_mdb_catalog",
    "error": {
      "code": 4,
      "codeName": "NoSuchKey",
      "errmsg": "Unable to find metadata for table:_mdb_catalog"
    }
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.945-06:00"
  },
  "s": "F",
  "c": "-",
  "id": 23095,
  "ctx": "initandlisten",
  "msg": "Fatal assertion",
  "attr": {
    "msgid": 50926,
    "error": "NoSuchKey: Unable to find metadata for table:_mdb_catalog",
    "file": "src\\mongo\\db\\storage\\storage_engine_impl.cpp",
    "line": 122
  }
}{
  "t": {
    "$date": "2022-03-20T13:45:27.951-06:00"
  },
  "s": "F",
  "c": "-",
  "id": 23096,
  "ctx": "initandlisten",
  "msg": "\n\n***aborting after fassert() failure\n\n"
}

        1. 1JpmD.png
          1JpmD.png
          132 kB

            Assignee:
            edwin.zhou@mongodb.com Edwin Zhou
            Reporter:
            pankaj@srashtasoft.com Pankaj Sakariya
            Votes:
            0 Vote for this issue
            Watchers:
            8 Start watching this issue

              Created:
              Updated:
              Resolved: