... Fri Jul 31 16:39:52 UTC 2015 Completed loop 20 out of 100, test time is 2723 Fri Jul 31 16:39:52 UTC 2015 Connect to server to install/recover mongod Fri Jul 31 16:39:53 UTC 2015 System was last booted 2015-07-31 12:39, up 53 seconds Fri Jul 31 16:39:53 UTC 2015 /home/jonathan/pt-317-wt-lsm2-100k/mongodb-linux-x86_64-3.1.7-pre-/bin/mongod --storageEngine wiredTiger --logpath pt-317-wt-lsm2-100k/logs/mongod-wiredTiger-recovery.log --logappend --dbpath pt-317-wt-lsm2-100k/data/wiredTiger --port 37017 --fork --setParameter enableTestCommands=1 --wiredTigerCollectionConfigString type=lsm --wiredTigerIndexConfigString type=lsm -replSet pc 2015-07-31T12:39:53.563-0400 I STORAGE [main] Collection custom option: type=lsm 2015-07-31T12:39:53.563-0400 I STORAGE [main] Index custom option: type=lsm about to fork child process, waiting until server is ready for connections. forked process: 1784 child process started successfully, parent exiting Fri Jul 31 16:40:01 UTC 2015 Mongod startup/recovery took 8 seconds Fri Jul 31 16:40:01 UTC 2015 echo function waitForPrimary(timeout, interval) { var timeStart = new Date().getTime(); var interval = interval || 500; var timeout = timeout || 60000; while (!db.isMaster().ismaster) { sleep(interval); var timeNow = new Date().getTime(); if (timeNow-timeStart > timeout) { return 0; } } return 1; }; rsconf = {_id: 'pc', members: [ {_id: 0, host: 'localhost:37017'}]}; if (db.getSisterDB('local').system.replset.count()) { rsconf = rs.conf(); rsconf.members[0].host = 'localhost:37017'; rs.reconfig(rsconf, {force: true}); print('reconfig', tojson(rsconf)); } else { rs.initiate(rsconf); print('initiate', tojson(rsconf)); } if (!waitForPrimary()) { print('Host did not become primary'); quit(1); }; rs.status(); db.createCollection('cycle') | /home/jonathan/pt-317-wt-lsm2-100k/mongodb-linux-x86_64-3.1.7-pre-/bin/mongo localhost:37017/power MongoDB shell version: 3.1.7-pre- connecting to: localhost:37017/power reconfig { "_id" : "pc", "version" : 2280940, "members" : [ { "_id" : 0, "host" : "localhost:37017", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : NumberLong(0), "votes" : 1 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 }, "protocolVersion" : NumberLong(0) } } { "ok" : 0, "errmsg" : "collection already exists", "code" : 48 } bye Connection to 10.4.124.130 closed. Fri Jul 31 16:40:10 UTC 2015 Connect to mongod and validate collection Fri Jul 31 16:40:10 UTC 2015 echo var val = db.cycle.validate(true); print(tojson(db.serverStatus()), tojson(val)); if (!val.valid) { print('db.cycle is invalid'); quit(1) } | /usr/local/bin/mongo 10.4.124.130:37017/power MongoDB shell version: 3.1.6 connecting to: 10.4.124.130:37017/power { "host" : "CAP-JA-Ubuntu14:37017", "version" : "3.1.7-pre-", "process" : "mongod", "pid" : NumberLong(1784), "uptime" : 41, "uptimeMillis" : NumberLong(41061), "uptimeEstimate" : 34, "localTime" : ISODate("2015-07-31T16:40:34.612Z"), "asserts" : { "regular" : 0, "warning" : 0, "msg" : 0, "user" : 0, "rollovers" : 0 }, "connections" : { "current" : 1, "available" : 818, "totalCreated" : NumberLong(2) }, "cursors" : { "note" : "deprecated, use server status metrics", "clientCursors_size" : 0, "totalOpen" : 0, "pinned" : 0, "totalNoTimeout" : 0, "timedOut" : 0 }, "extra_info" : { "note" : "fields vary by platform", "heap_usage_bytes" : 872281240, "page_faults" : 173 }, "globalLock" : { "totalTime" : NumberLong(41115000), "currentQueue" : { "total" : 0, "readers" : 0, "writers" : 0 }, "activeClients" : { "total" : 11, "readers" : 0, "writers" : 0 } }, "locks" : { "Global" : { "acquireCount" : { "r" : NumberLong(222), "w" : NumberLong(60), "R" : NumberLong(1), "W" : NumberLong(7) }, "acquireWaitCount" : { "r" : NumberLong(1), "w" : NumberLong(1) }, "timeAcquiringMicros" : { "r" : NumberLong(1340872), "w" : NumberLong(2085) } }, "Database" : { "acquireCount" : { "r" : NumberLong(74), "w" : NumberLong(38), "R" : NumberLong(3), "W" : NumberLong(22) } }, "Collection" : { "acquireCount" : { "r" : NumberLong(68), "W" : NumberLong(1) }, "acquireWaitCount" : { "r" : NumberLong(1) }, "timeAcquiringMicros" : { "r" : NumberLong(563705280) } }, "Metadata" : { "acquireCount" : { "w" : NumberLong(2) } }, "oplog" : { "acquireCount" : { "r" : NumberLong(6), "w" : NumberLong(37) } } }, "network" : { "bytesIn" : 1840, "bytesOut" : 5819, "numRequests" : 22 }, "opcounters" : { "insert" : 0, "query" : 1, "update" : 0, "delete" : 0, "getmore" : 0, "command" : 23 }, "opcountersRepl" : { "insert" : 0, "query" : 0, "update" : 0, "delete" : 0, "getmore" : 0, "command" : 0 }, "repl" : { "setName" : "pc", "setVersion" : 2302354, "ismaster" : true, "secondary" : false, "hosts" : [ "localhost:37017" ], "primary" : "localhost:37017", "me" : "localhost:37017", "electionId" : ObjectId("55bba4e40000000000000000"), "rbid" : 443721945 }, "storageEngine" : { "name" : "wiredTiger", "supportsCommittedReads" : true }, "wiredTiger" : { "uri" : "statistics:", "LSM" : { "sleep for LSM checkpoint throttle" : 0, "sleep for LSM merge throttle" : 0, "rows merged in an LSM tree" : 0, "application work units currently queued" : 0, "merge work units currently queued" : 0, "tree queue hit maximum" : 2015, "switch work units currently queued" : 0, "tree maintenance operations scheduled" : 49905, "tree maintenance operations discarded" : 0, "tree maintenance operations executed" : 49881 }, "async" : { "number of allocation state races" : 0, "number of operation slots viewed for allocation" : 0, "current work queue length" : 0, "number of flush calls" : 0, "number of times operation allocation failed" : 0, "maximum work queue length" : 0, "number of times worker found no work" : 0, "total allocations" : 0, "total compact calls" : 0, "total insert calls" : 0, "total remove calls" : 0, "total search calls" : 0, "total update calls" : 0 }, "block-manager" : { "mapped bytes read" : 455856128, "bytes read" : 453390336, "bytes written" : 1085440, "mapped blocks read" : 90895, "blocks pre-loaded" : 2018, "blocks read" : 90408, "blocks written" : 127 }, "cache" : { "tracked dirty bytes in the cache" : 369801, "tracked bytes belonging to internal pages in the cache" : 4466746, "bytes currently in the cache" : 858986047, "tracked bytes belonging to leaf pages in the cache" : 854519301, "maximum bytes configured" : 1073741824, "tracked bytes belonging to overflow pages in the cache" : 0, "bytes read into cache" : NumberLong("63314259612831"), "bytes written from cache" : 1153102, "pages evicted by application threads" : 0, "checkpoint blocked page eviction" : 0, "unmodified pages evicted" : 108465, "page split during eviction deepened the tree" : 0, "modified pages evicted" : 33, "pages selected for eviction unable to be evicted" : 290303, "pages evicted because they exceeded the in-memory maximum" : 0, "pages evicted because they had chains of deleted items" : 2, "failed eviction of pages that exceeded the in-memory maximum" : 340, "hazard pointer blocked page eviction" : 4435, "internal pages evicted" : 337, "maximum page size at eviction" : 249657, "eviction server candidate queue empty when topping up" : 3040, "eviction server candidate queue not empty when topping up" : 3428, "eviction server evicting pages" : 248, "eviction server populating queue, but not evicting pages" : 6462, "eviction server unable to reach eviction goal" : 0, "pages split during eviction" : 7, "pages walked for eviction" : 26497777, "eviction worker thread evicting pages" : 403137, "in-memory page splits" : 0, "percentage overhead" : 8, "tracked dirty pages in the cache" : 18, "pages currently held in the cache" : 28067, "pages read into cache" : 181276, "pages written from cache" : 94 }, "connection" : { "pthread mutex condition wait calls" : 33671, "files currently open" : 215, "memory allocations" : 662326, "memory frees" : 527876, "memory re-allocations" : 240, "total read I/Os" : 144003, "pthread mutex shared lock read-lock calls" : 21692, "pthread mutex shared lock write-lock calls" : 10195, "total write I/Os" : 246 }, "cursor" : { "cursor create calls" : 1568, "cursor insert calls" : 24786, "cursor next calls" : 18042659, "cursor prev calls" : 152, "cursor remove calls" : 280, "cursor reset calls" : 25770, "cursor search calls" : 3019, "cursor search near calls" : 50, "cursor update calls" : 1 }, "data-handle" : { "connection dhandles swept" : 2, "connection candidate referenced" : 5, "connection sweeps" : 4, "connection time-of-death sets" : 150, "session dhandles swept" : 0, "session sweep attempts" : 35 }, "log" : { "total log buffer size" : 33554432, "log bytes of payload data" : 17338, "log bytes written" : 26496, "yields waiting for previous log file close" : 0, "total size of compressed records" : 12798, "total in-memory size of compressed records" : 18750, "log records too small to compress" : 87, "log records not compressed" : 1, "log records compressed" : 20, "maximum log file size" : 104857600, "pre-allocated log files prepared" : 2, "number of pre-allocated log files to create" : 2, "pre-allocated log files used" : 0, "log release advances write LSN" : 108, "records processed by log scan" : 52849, "log scan records requiring two reads" : 483, "log scan operations" : 3, "consolidated slot closures" : 0, "written slots coalesced" : 0, "logging bytes consolidated" : 0, "consolidated slot joins" : 0, "consolidated slot join races" : 0, "record size exceeded maximum" : 0, "failed to find a slot large enough for record" : 0, "consolidated slot join transitions" : 0, "log sync operations" : 29, "log sync_dir operations" : 1, "log server thread advances write LSN" : 0, "log write operations" : 108 }, "reconciliation" : { "page reconciliation calls" : 77, "page reconciliation calls for eviction" : 6, "split bytes currently awaiting free" : 567, "split objects currently awaiting free" : 21 }, "session" : { "open cursor count" : 116, "open session count" : 34 }, "thread-yield" : { "page acquire busy blocked" : 1, "page acquire eviction blocked" : 0, "page acquire locked blocked" : 0, "page acquire read blocked" : 0, "page acquire time sleeping (usecs)" : 0 }, "transaction" : { "transaction begins" : 50, "transaction checkpoints" : 1, "transaction checkpoint generation" : 1, "transaction checkpoint currently running" : 0, "transaction checkpoint max time (msecs)" : 787, "transaction checkpoint min time (msecs)" : 787, "transaction checkpoint most recent time (msecs)" : 787, "transaction checkpoint total time (msecs)" : 787, "transactions committed" : 5, "transaction failures due to cache overflow" : 0, "transaction range of IDs currently pinned by a checkpoint" : 0, "transaction range of IDs currently pinned" : 0, "transactions rolled back" : 44, "transaction sync calls" : 0 }, "concurrentTransactions" : { "write" : { "out" : 0, "available" : 128, "totalTickets" : 128 }, "read" : { "out" : 1, "available" : 127, "totalTickets" : 128 } } }, "writeBacksQueued" : false, "mem" : { "bits" : 64, "resident" : 1376, "virtual" : 1931, "supported" : true, "mapped" : 0, "mappedWithJournal" : 0 }, "metrics" : { "commands" : { "count" : { "failed" : NumberLong(0), "total" : NumberLong(1) }, "create" : { "failed" : NumberLong(1), "total" : NumberLong(1) }, "isMaster" : { "failed" : NumberLong(0), "total" : NumberLong(12) }, "replSetGetConfig" : { "failed" : NumberLong(0), "total" : NumberLong(2) }, "replSetGetStatus" : { "failed" : NumberLong(1), "total" : NumberLong(3) }, "replSetReconfig" : { "failed" : NumberLong(0), "total" : NumberLong(1) }, "serverStatus" : { "failed" : NumberLong(0), "total" : NumberLong(1) }, "validate" : { "failed" : NumberLong(0), "total" : NumberLong(1) }, "whatsmyuri" : { "failed" : NumberLong(0), "total" : NumberLong(1) } }, "cursor" : { "timedOut" : NumberLong(0), "open" : { "noTimeout" : NumberLong(0), "pinned" : NumberLong(0), "total" : NumberLong(0) } }, "document" : { "deleted" : NumberLong(0), "inserted" : NumberLong(0), "returned" : NumberLong(0), "updated" : NumberLong(0) }, "getLastError" : { "wtime" : { "num" : 0, "totalMillis" : 0 }, "wtimeouts" : NumberLong(0) }, "operation" : { "fastmod" : NumberLong(0), "idhack" : NumberLong(0), "scanAndOrder" : NumberLong(0), "writeConflicts" : NumberLong(0) }, "queryExecutor" : { "scanned" : NumberLong(0), "scannedObjects" : NumberLong(0) }, "record" : { "moves" : NumberLong(0) }, "repl" : { "apply" : { "batches" : { "num" : 0, "totalMillis" : 0 }, "ops" : NumberLong(0) }, "buffer" : { "count" : NumberLong(0), "maxSizeBytes" : 268435456, "sizeBytes" : NumberLong(0) }, "network" : { "bytes" : NumberLong(0), "getmores" : { "num" : 0, "totalMillis" : 0 }, "ops" : NumberLong(0), "readersCreated" : NumberLong(6) }, "preload" : { "docs" : { "num" : 0, "totalMillis" : 0 }, "indexes" : { "num" : 0, "totalMillis" : 0 } } }, "storage" : { "freelist" : { "search" : { "bucketExhausted" : NumberLong(0), "requests" : NumberLong(0), "scanned" : NumberLong(0) } } }, "ttl" : { "deletedDocuments" : NumberLong(0), "passes" : NumberLong(0) } }, "ok" : 1 } { "ns" : "power.cycle", "nrecords" : 5849143, "nIndexes" : 2, "keysPerIndex" : { "power.cycle.$_id_" : 5849143, "power.cycle.$x_1" : 5849143 }, "indexDetails" : { "power.cycle.$_id_" : { "valid" : true }, "power.cycle.$x_1" : { "valid" : true } }, "valid" : true, "errors" : [ ], "ok" : 1 } bye Fri Jul 31 16:40:39 UTC 2015 Restart mongod on standard port Fri Jul 31 16:40:40 UTC 2015 echo db.adminCommand({shutdown:1, force: true}) | /home/jonathan/pt-317-wt-lsm2-100k/mongodb-linux-x86_64-3.1.7-pre-/bin/mongo localhost:37017/power MongoDB shell version: 3.1.7-pre- connecting to: localhost:37017/power 2015-07-31T12:40:40.358-0400 E QUERY [thread1] Error: error doing query: failed : DB.prototype.runCommand@src/mongo/shell/db.js:124:1 DB.prototype.adminCommand@src/mongo/shell/db.js:138:12 @(shell):1:1 2015-07-31T12:40:40.360-0400 I NETWORK [thread1] trying reconnect to localhost:37017 (127.0.0.1) failed 2015-07-31T12:40:40.540-0400 I NETWORK [thread1] Socket recv() errno:104 Connection reset by peer 127.0.0.1:37017 2015-07-31T12:40:40.540-0400 I NETWORK [thread1] SocketException: remote: 127.0.0.1:37017 error: 9001 socket exception [RECV_ERROR] server [127.0.0.1:37017] 2015-07-31T12:40:40.540-0400 I NETWORK [thread1] reconnect localhost:37017 (127.0.0.1) failed failed 2015-07-31T12:40:40.543-0400 I NETWORK [thread1] trying reconnect to localhost:37017 (127.0.0.1) failed 2015-07-31T12:40:40.543-0400 W NETWORK [thread1] Failed to connect to 127.0.0.1:37017, reason: errno:111 Connection refused 2015-07-31T12:40:40.544-0400 I NETWORK [thread1] reconnect localhost:37017 (127.0.0.1) failed failed bye Fri Jul 31 16:40:45 UTC 2015 Waiting for mongod to release pt-317-wt-lsm2-100k/data/wiredTiger/mongod.lock Fri Jul 31 16:40:45 UTC 2015 /home/jonathan/pt-317-wt-lsm2-100k/mongodb-linux-x86_64-3.1.7-pre-/bin/mongod --storageEngine wiredTiger --logpath pt-317-wt-lsm2-100k/logs/mongod-wiredTiger.log --logappend --dbpath pt-317-wt-lsm2-100k/data/wiredTiger --port 27017 --fork --setParameter enableTestCommands=1 --wiredTigerCollectionConfigString type=lsm --wiredTigerIndexConfigString type=lsm -replSet pc 2015-07-31T12:40:45.590-0400 I STORAGE [main] Collection custom option: type=lsm 2015-07-31T12:40:45.590-0400 I STORAGE [main] Index custom option: type=lsm about to fork child process, waiting until server is ready for connections. forked process: 4574 child process started successfully, parent exiting Fri Jul 31 17:00:49 UTC 2015 Mongod startup/recovery took 1204 seconds Fri Jul 31 17:00:49 UTC 2015 echo rsconf = rs.conf(); rsconf.members[0].host = 'localhost:27017'; rs.reconfig(rsconf, {force: true}); function waitForPrimary(timeout, interval) { var timeStart = new Date().getTime(); var interval = interval || 500; var timeout = timeout || 60000; while (!db.isMaster().ismaster) { sleep(interval); var timeNow = new Date().getTime(); if (timeNow-timeStart > timeout) { return 0; } } return 1; }; if (!waitForPrimary()) { print('Host did not become primary'); quit(1); }; rs.status(); db.serverStatus() | /home/jonathan/pt-317-wt-lsm2-100k/mongodb-linux-x86_64-3.1.7-pre-/bin/mongo localhost:27017/power MongoDB shell version: 3.1.7-pre- connecting to: localhost:27017/power 2015-07-31T13:00:49.312-0400 W NETWORK [thread1] Failed to connect to 127.0.0.1:27017, reason: errno:111 Connection refused 2015-07-31T13:00:49.312-0400 E QUERY [thread1] Error: couldn't connect to server localhost:27017, connection attempt failed : connect@src/mongo/shell/mongo.js:216:14 @(connect):1:6 exception: connect failed Fri Jul 31 17:00:54 UTC 2015 echo rsconf = rs.conf(); rsconf.members[0].host = 'localhost:27017'; rs.reconfig(rsconf, {force: true}); function waitForPrimary(timeout, interval) { var timeStart = new Date().getTime(); var interval = interval || 500; var timeout = timeout || 60000; while (!db.isMaster().ismaster) { sleep(interval); var timeNow = new Date().getTime(); if (timeNow-timeStart > timeout) { return 0; } } return 1; }; if (!waitForPrimary()) { print('Host did not become primary'); quit(1); }; rs.status(); db.serverStatus() | /home/jonathan/pt-317-wt-lsm2-100k/mongodb-linux-x86_64-3.1.7-pre-/bin/mongo localhost:27017/power MongoDB shell version: 3.1.7-pre- connecting to: localhost:27017/power 2015-07-31T13:00:54.387-0400 W NETWORK [thread1] Failed to connect to 127.0.0.1:27017, reason: errno:111 Connection refused 2015-07-31T13:00:54.388-0400 E QUERY [thread1] Error: couldn't connect to server localhost:27017, connection attempt failed : connect@src/mongo/shell/mongo.js:216:14 @(connect):1:6 exception: connect failed Connection to 10.4.124.130 closed. Fri Jul 31 17:00:59 UTC 2015 Unable to connect to 10.4.124.130 Fri Jul 31 17:00:59 UTC 2015 Exiting powerTest Fri Jul 31 17:00:59 UTC 2015 Cleaning up mongo clients logs