-
Type: Bug
-
Resolution: Done
-
Priority: Major - P3
-
None
-
Affects Version/s: 3.4.5
-
Component/s: WiredTiger
-
ALL
I am using mongodb version v3.4.5, Storage engine - WiredTiger on
Mongo db sever not starting due to unexpected shutdown.
My issue is same as https://jira.mongodb.org/browse/SERVER-18448 and https://jira.mongodb.org/browse/SERVER-30942
Dbpath = /var/lib/mongo
OS = CentOS Linux 7 (Core) 64bit on Vmware
Disk type = HDD
Disk management = LVM
Disk is working fine.
RAID = RAID not configured.
I have used below --repair order to repair it, this error message show. I have attached other required information with WiredTiger files. Can you help me fix this problem?
[root@centos7 /]# mongod --repair --dbpath /var/lib/mongo 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] MongoDB starting : pid=3404 port=27017 dbpath=/var/lib/mongo 64-bit host=centos7 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] db version v3.4.5 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] git version: 520b8f3092c48d934f0cd78ab5f40fe594f96863 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] OpenSSL version: OpenSSL 1.0.1e-fips 11 Feb 2013 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] allocator: tcmalloc 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] modules: none 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] build environment: 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] distmod: rhel70 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] distarch: x86_64 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] target_arch: x86_64 2020-01-22T15:51:58.145+0530 I CONTROL [initandlisten] options: { repair: true, storage: { dbPath: "/var/lib/mongo" } } 2020-01-22T15:51:58.181+0530 I - [initandlisten] Detected data files in /var/lib/mongo created by the 'wiredTiger' storage engine, so setting the active storage engine to 'wiredTiger'. 2020-01-22T15:51:58.181+0530 I STORAGE [initandlisten] Detected WT journal files. Running recovery from last checkpoint. 2020-01-22T15:51:58.181+0530 I STORAGE [initandlisten] journal to nojournal transition config: create,cache_size=899M,session_max=20000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0), 2020-01-22T15:51:58.219+0530 E STORAGE [initandlisten] WiredTiger error (0) [1579688518:219342][3404:0x7f2160642e40], file:WiredTiger.wt, WT_CURSOR.remove: read checksum error for 32768B block at offset 1777664: block header checksum of 658458978 doesn't match expected checksum of 1085055468 2020-01-22T15:51:58.219+0530 E STORAGE [initandlisten] WiredTiger error (0) [1579688518:219422][3404:0x7f2160642e40], file:WiredTiger.wt, WT_CURSOR.remove: WiredTiger.wt: encountered an illegal file format or internal value 2020-01-22T15:51:58.219+0530 E STORAGE [initandlisten] WiredTiger error (-31804) [1579688518:219444][3404:0x7f2160642e40], file:WiredTiger.wt, WT_CURSOR.remove: the process must exit and restart: WT_PANIC: WiredTiger library panic 2020-01-22T15:51:58.219+0530 I - [initandlisten] Fatal Assertion 28558 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 361 2020-01-22T15:51:58.219+0530 I - [initandlisten] ***aborting after fassert() failure 2020-01-22T15:51:58.264+0530 F - [initandlisten] Got signal: 6 (Aborted). 0x7f2161bc8de1 0x7f2161bc7ff9 0x7f2161bc84dd 0x7f215f2ac370 0x7f215ef111d7 0x7f215ef128c8 0x7f2160e71053 0x7f21618db6a6 0x7f2160e7b362 0x7f2160e7b57e 0x7f2160e7b7d6 0x7f21624d33e5 0x7f21624ed16b 0x7f21624f42c5 0x7f21625151a0 0x7f21624e589c 0x7f21625325ca 0x7f21625c2604 0x7f2162558840 0x7f21625c3274 0x7f21625286e7 0x7f2162520912 0x7f21618bf9f0 0x7f21618b8755 0x7f21617ab407 0x7f2160e5d22c 0x7f2160e7cb8b 0x7f215eefdb35 0x7f2160edb37f ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"7F2160657000","o":"1571DE1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"7F2160657000","o":"1570FF9"},{"b":"7F2160657000","o":"15714DD"},{"b":"7F215F29D000","o":"F370"},{"b":"7F215EEDC000","o":"351D7","s":"gsignal"},{"b":"7F215EEDC000","o":"368C8","s":"abort"},{"b":"7F2160657000","o":"81A053","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"7F2160657000","o":"12846A6"},{"b":"7F2160657000","o":"824362","s":"__wt_eventv"},{"b":"7F2160657000","o":"82457E","s":"__wt_err"},{"b":"7F2160657000","o":"8247D6","s":"__wt_panic"},{"b":"7F2160657000","o":"1E7C3E5","s":"__wt_bm_read"},{"b":"7F2160657000","o":"1E9616B","s":"__wt_bt_read"},{"b":"7F2160657000","o":"1E9D2C5","s":"__wt_page_in_func"},{"b":"7F2160657000","o":"1EBE1A0","s":"__wt_row_search"},{"b":"7F2160657000","o":"1E8E89C","s":"__wt_btcur_remove"},{"b":"7F2160657000","o":"1EDB5CA"},{"b":"7F2160657000","o":"1F6B604"},{"b":"7F2160657000","o":"1F01840","s":"__wt_log_scan"},{"b":"7F2160657000","o":"1F6C274","s":"__wt_txn_recover"},{"b":"7F2160657000","o":"1ED16E7","s":"__wt_connection_workers"},{"b":"7F2160657000","o":"1EC9912","s":"wiredtiger_open"},{"b":"7F2160657000","o":"12689F0","s":"_ZN5mongo18WiredTigerKVEngineC2ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_PNS_11ClockSourceES8_mbbbb"},{"b":"7F2160657000","o":"1261755"},{"b":"7F2160657000","o":"1154407","s":"_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv"},{"b":"7F2160657000","o":"80622C"},{"b":"7F2160657000","o":"825B8B","s":"main"},{"b":"7F215EEDC000","o":"21B35","s":"__libc_start_main"},{"b":"7F2160657000","o":"88437F"}],"processInfo":{ "mongodbVersion" : "3.4.5", "gitVersion" : "520b8f3092c48d934f0cd78ab5f40fe594f96863", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-514.21.1.el7.x86_64", "version" : "#1 SMP Thu May 25 17:04:51 UTC 2017", "machine" : "x86_64" }, "somap" : [ { "b" : "7F2160657000", "elfType" : 3, "buildId" : "2611E67268F802B66C2DEAD16C73B6336CED2F25" }, { "b" : "7FFCE26C6000", "elfType" : 3, "buildId" : "6383687783BD78FDBE923F025C6D284C926E7E15" }, { "b" : "7F21601C7000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "90EAF65D9B0EEEB1424241281F7F197451D4317D" }, { "b" : "7F215FDDD000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "7278C69EE161D98DDD0FA00F92B67AD78C7B7F40" }, { "b" : "7F215FBD5000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "82E77ADE22BC9FFF8D3458BD37331E7EDF174C28" }, { "b" : "7F215F9D1000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "C5F560504E1AF52E29679C3B52FF11121015D6BB" }, { "b" : "7F215F6CF000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "721C7CC9488EFA25F83B48AF713AB27DBE48EF3E" }, { "b" : "7F215F4B9000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "408B46E291B2D4C9612E27C0509D165D7E186D40" }, { "b" : "7F215F29D000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "C3DEB1FA27CD0C1C3CC575B944ABACBA0698B0F2" }, { "b" : "7F215EEDC000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "2887288BE2E665C31B1C2DA3F1E8AE74631B25AA" }, { "b" : "7F2160435000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "8F3E366E2DB73C330A3791DEAE31AE9579099B44" }, { "b" : "7F215EC8E000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "A2499C359AA179EE23324ED949C0E508E4434F10" }, { "b" : "7F215E9A7000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "E09A34D9083DC6FEAF7018C09D55631DEEE2836D" }, { "b" : "7F215E7A3000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "BF54B7C8932E450769FBBB8B18864D1DD70BBC67" }, { "b" : "7F215E571000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "BF8F00D7CB849ADB0B7A4703BC7B8D66AEE6A49C" }, { "b" : "7F215E35B000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "EA8E45DC8E395CC5E26890470112D97A1F1E0B65" }, { "b" : "7F215E14C000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "1E7A92FDD6FB3871DA97F4BCA2E147E72B6B6E1F" }, { "b" : "7F215DF48000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7F215DD2E000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "FE7AE845A123A3DFC0FDC2408BCBC2BA8B61B158" }, { "b" : "7F215DB07000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "76687CA31A406854DF3BCF8D03055656F56E6892" }, { "b" : "7F215D8A6000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "AE64AA461A26E01F60408013D361749D56DD0AE1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x7f2161bc8de1] mongod(+0x1570FF9) [0x7f2161bc7ff9] mongod(+0x15714DD) [0x7f2161bc84dd] libpthread.so.0(+0xF370) [0x7f215f2ac370] libc.so.6(gsignal+0x37) [0x7f215ef111d7] libc.so.6(abort+0x148) [0x7f215ef128c8] mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x7f2160e71053] mongod(+0x12846A6) [0x7f21618db6a6] mongod(__wt_eventv+0x3D7) [0x7f2160e7b362] mongod(__wt_err+0x9D) [0x7f2160e7b57e] mongod(__wt_panic+0x24) [0x7f2160e7b7d6] mongod(__wt_bm_read+0x135) [0x7f21624d33e5] mongod(__wt_bt_read+0x1FB) [0x7f21624ed16b] mongod(__wt_page_in_func+0x1275) [0x7f21624f42c5] mongod(__wt_row_search+0x660) [0x7f21625151a0] mongod(__wt_btcur_remove+0xE2C) [0x7f21624e589c] mongod(+0x1EDB5CA) [0x7f21625325ca] mongod(+0x1F6B604) [0x7f21625c2604] mongod(__wt_log_scan+0xC80) [0x7f2162558840] mongod(__wt_txn_recover+0x3A4) [0x7f21625c3274] mongod(__wt_connection_workers+0x37) [0x7f21625286e7] mongod(wiredtiger_open+0x1672) [0x7f2162520912] mongod(_ZN5mongo18WiredTigerKVEngineC2ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_PNS_11ClockSourceES8_mbbbb+0xD20) [0x7f21618bf9f0] mongod(+0x1261755) [0x7f21618b8755] mongod(_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv+0x697) [0x7f21617ab407] mongod(+0x80622C) [0x7f2160e5d22c] mongod(main+0x96B) [0x7f2160e7cb8b] libc.so.6(__libc_start_main+0xF5) [0x7f215eefdb35] mongod(+0x88437F) [0x7f2160edb37f] ----- END BACKTRACE ----- Aborted