Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-33720

Failure: jsCore on Ubuntu 14.04 (RocksDB) (repair_database.js) [MongoDB (3.6) @ e1cc0fb4]

    • Type: Icon: Bug Bug
    • Resolution: Won't Fix
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: None
    • Storage Execution
    • ALL
    • 45

      jsCore failed on Ubuntu 14.04 (RocksDB)

      Host: ec2-54-82-137-152.compute-1.amazonaws.com
      Project: MongoDB (3.6)
      Commit: diff: SERVER-32604 prevent adding a binary last-stable shard if the cluster is upgrading or downgrading
      repair_database.js - Logs | History

      Found a definite cause of failure for jstests/core/repair_database.js

      2018-02-23T18:13:47.302000: Javascript StackTrace

      Logs:

      Error: error doing query: failed: network error while attempting to run command 'repairDatabase' on host 'localhost:20500'  :
      

      Stacktrace:

      DB.prototype.runCommand@src/mongo/shell/db.js:168:1
      DB.prototype.repairDatabase@src/mongo/shell/db.js:584:16
      @jstests/core/repair_database.js:40:22
      

      Here are some events that may help with triaging the failure

      2018-02-23T18:13:47.231000: 2018-02-23T18:13:47.230+0000 I STORAGE [conn52] Repairing collection repairDB.a

      Logs:

      2018-02-23T18:13:47.230+0000 I STORAGE [conn52] Repairing collection repairDB.a
      2018-02-23T18:13:47.231+0000 F - [conn52] Invariant failure slice.size() == sizeof(int64_t) src/mongo/db/modules/rocksdb/src/rocks_record_store.cpp 1008
      2018-02-23T18:13:47.231+0000 F - [conn52] 
      ***aborting after invariant() failure
      
      
      2018-02-23T18:13:47.260+0000 F - [conn52] Got signal: 6 (Aborted).
      
      0x24628e8 0x2461d2d 0x2462134 0x7f96a7f07330 0x7f96a7b64c37 0x7f96a7b68028 0xbf5dae 0xf1ef78 0xf24c35 0xf1ab2a 0xefe4bf 0xef7dcb 0x11878b7 0xf90728 0x1e10f06 0x1e12217 0x1e0f1bf 0xf50cbe 0xf52ebf 0xf53cf0 0xf5fcef 0xf5bb87 0xf5ec5a 0x1dd0042 0xf5aae5 0xf5cef2 0xf5d801 0xf5bbf5 0xf5ec5a 0x1dd05c1 0x23381b4 0x7f96a7eff184 0x7f96a7c2c03d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"20628E8","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"2061D2D"},{"b":"400000","o":"2062134"},{"b":"7F96A7EF7000","o":"10330"},{"b":"7F96A7B2E000","o":"36C37","s":"gsignal"},{"b":"7F96A7B2E000","o":"3A028","s":"abort"},{"b":"400000","o":"7F5DAE","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"400000","o":"B1EF78"},{"b":"400000","o":"B24C35","s":"_ZN5mongo16RocksRecordStoreC1ENS_10StringDataES1_PN7rocksdb2DBEPNS_19RocksCounterManagerEPNS_22RocksDurabilityManagerEPNS_24RocksCompactionSchedulerENSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbllPNS_14CappedCallbackE"},{"b":"400000","o":"B1AB2A","s":"_ZN5mongo11RocksEngine14getRecordStoreEPNS_16OperationContextENS_10StringDataES3_RKNS_17CollectionOptionsE"},{"b":"400000","o":"AFE4BF","s":"_ZN5mongo26KVDatabaseCatalogEntryBase14initCollectionEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb"},{"b":"400000","o":"AF7DCB","s":"_ZN5mongo15KVStorageEngine17repairRecordStoreEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE"},{"b":"400000","o":"D878B7","s":"_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbb"},{"b":"400000","o":"B90728","s":"_ZN5mongo17CmdRepairDatabase9errmsgRunEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERS8_RNS_14BSONObjBuilderE"},{"b":"400000","o":"1A10F06","s":"_ZN5mongo23ErrmsgCommandDeprecated3runEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE"},{"b":"400000","o":"1A12217","s":"_ZN5mongo12BasicCommand11enhancedRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE"},{"b":"400000","o":"1A0F1BF","s":"_ZN5mongo7Command9publicRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE"},{"b":"400000","o":"B50CBE"},{"b":"400000","o":"B52EBF"},{"b":"400000","o":"B53CF0","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"400000","o":"B5FCEF","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"400000","o":"B5BB87","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"400000","o":"B5EC5A"},{"b":"400000","o":"19D0042","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsE"},{"b":"400000","o":"B5AAE5","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS0_9OwnershipE"},{"b":"400000","o":"B5CEF2","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"400000","o":"B5D801","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"400000","o":"B5BBF5","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"400000","o":"B5EC5A"},{"b":"400000","o":"19D05C1"},{"b":"400000","o":"1F381B4"},{"b":"7F96A7EF7000","o":"8184"},{"b":"7F96A7B2E000","o":"FE03D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.6.3-2-ge1cc0fb", "gitVersion" : "e1cc0fb45f6b5bbe596158258122d188667a43fe", "compiledModules" : [ "rocksdb" ], "uname" : { "sysname" : "Linux", "release" : "3.13.0-24-generic", "version" : "#46-Ubuntu SMP Thu Apr 10 19:11:08 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E2B68AC9F64C81DEB87F0DBA0BB1FD9E47624ED3" }, { "b" : "7FFFDB8FE000", "elfType" : 3, "buildId" : "6755FAD2CADACDF1667E5B57FF1EDFC28DD1C976" }, { "b" : "7F96A8E68000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "7EE9BC42787A0F9A793BDBFCB0671FD12C52BC2A" }, { "b" : "7F96A8C58000", "path" : "/lib/x86_64-linux-gnu/libbz2.so.1.0", "elfType" : 3, "buildId" : "E1031DDBFFE20367E874B7093EEC0C8D9F3B43F6" }, { "b" : "7F96A8A3D000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "AD304AFCE6847F7A4D66D22853E87CCBF5A66966" }, { "b" : "7F96A8835000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "4F930712D3609C93E380E5BE5DF73E7AD273531C" }, { "b" : "7F96A8631000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "034D6A4EE9DCAB4A34ABD644345CBBB42DC63088" }, { "b" : "7F96A832B000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "300C7884CDEB5667BEA2357D2B8E7A76397562D6" }, { "b" : "7F96A8115000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF00791DED7359DBB92" }, { "b" : "7F96A7EF7000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "F64B8AD471FBA1B7A3A64EFB01551E694975E1F7" }, { "b" : "7F96A7B2E000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "D9A10B8EF90300628DD0A3A535106967714D7328" }, { "b" : "7F96A9081000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "2CA513EDC89C7BC06EC183D1A3A03CC0F606319C" } ] }}
      mongod(_ZN5mongo15printStackTraceERSo+0x28) [0x24628e8]
      mongod(+0x2061D2D) [0x2461d2d]
      mongod(+0x2062134) [0x2462134]
      libpthread.so.0(+0x10330) [0x7f96a7f07330]
      libc.so.6(gsignal+0x37) [0x7f96a7b64c37]
      libc.so.6(abort+0x148) [0x7f96a7b68028]
      mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0x0) [0xbf5dae]
      mongod(+0xB1EF78) [0xf1ef78]
      mongod(_ZN5mongo16RocksRecordStoreC1ENS_10StringDataES1_PN7rocksdb2DBEPNS_19RocksCounterManagerEPNS_22RocksDurabilityManagerEPNS_24RocksCompactionSchedulerENSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbllPNS_14CappedCallbackE+0xD05) [0xf24c35]
      mongod(_ZN5mongo11RocksEngine14getRecordStoreEPNS_16OperationContextENS_10StringDataES3_RKNS_17CollectionOptionsE+0x82A) [0xf1ab2a]
      mongod(_ZN5mongo26KVDatabaseCatalogEntryBase14initCollectionEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb+0x2AF) [0xefe4bf]
      mongod(_ZN5mongo15KVStorageEngine17repairRecordStoreEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE+0x1DB) [0xef7dcb]
      mongod(_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbb+0x5F7) [0x11878b7]
      mongod(_ZN5mongo17CmdRepairDatabase9errmsgRunEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERS8_RNS_14BSONObjBuilderE+0x298) [0xf90728]
      mongod(_ZN5mongo23ErrmsgCommandDeprecated3runEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE+0x36) [0x1e10f06]
      mongod(_ZN5mongo12BasicCommand11enhancedRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE+0x67) [0x1e12217]
      mongod(_ZN5mongo7Command9publicRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE+0x1F) [0x1e0f1bf]
      mongod(+0xB50CBE) [0xf50cbe]
      mongod(+0xB52EBF) [0xf52ebf]
      mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x290) [0xf53cf0]
      mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0x9F) [0xf5fcef]
      mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x77) [0xf5bb87]
      mongod(+0xB5EC5A) [0xf5ec5a]
      mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsE+0x182) [0x1dd0042]
      mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS0_9OwnershipE+0x125) [0xf5aae5]
      mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0xAF2) [0xf5cef2]
      mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x201) [0xf5d801]
      mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0xE5) [0xf5bbf5]
      mongod(+0xB5EC5A) [0xf5ec5a]
      mongod(+0x19D05C1) [0x1dd05c1]
      mongod(+0x1F381B4) [0x23381b4]
      libpthread.so.0(+0x8184) [0x7f96a7eff184]
      libc.so.6(clone+0x6D) [0x7f96a7c2c03d]
      -----  END BACKTRACE  -----
      mongod on port 20500 was expected to be running in _do_teardown(), but wasn't. Exited with code -6.
      

      JavaScript Backtrace Context

      Frame 44: jstests/core/repair_database.js:40

      repair_database.js
       036     // 4
       037     var myColl2 = mydb.b;
       038     myColl.insert(doc);
       039     myColl2.insert(doc);
       040 >>> assert.commandWorked(mydb.repairDatabase());
       041     assert.docEq(doc, myColl.findOne({a: doc.a}));
       042     assert.docEq(doc, myColl2.findOne({a: doc.a}));
       043     
       044     // 5
       045     var ldb = db.getSisterDB("local");
      

            Assignee:
            backlog-server-execution [DO NOT USE] Backlog - Storage Execution Team
            Reporter:
            xgen-evg-user Xgen-Evergreen-User
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Created:
              Updated:
              Resolved: