cwd [/home/gregorv/Workspaces/10Gen/mongo] num procs:220 removing: /data/db/sconsTests//local.ns removing: /data/db/sconsTests//mongod.lock removing: /data/db/sconsTests//local.0 Fri Jan 18 13:15:25.032 [initandlisten] MongoDB starting : pid=23806 port=27999 dbpath=/data/db/sconsTests/ 64-bit host=nuwen Fri Jan 18 13:15:25.033 [initandlisten] Fri Jan 18 13:15:25.033 [initandlisten] ** NOTE: This is a development version (2.3.3-pre-) of MongoDB. Fri Jan 18 13:15:25.033 [initandlisten] ** Not recommended for production. Fri Jan 18 13:15:25.033 [initandlisten] Fri Jan 18 13:15:25.033 [initandlisten] db version v2.3.3-pre-, pdfile version 4.5 Fri Jan 18 13:15:25.033 [initandlisten] git version: d3e8692ad5981bba5eaa50966ba0baf3147e7d9f Fri Jan 18 13:15:25.033 [initandlisten] build info: Linux nuwen 3.5.0-21-generic #32-Ubuntu SMP Tue Dec 11 18:51:59 UTC 2012 x86_64 BOOST_LIB_VERSION=1_49 Fri Jan 18 13:15:25.033 [initandlisten] allocator: tcmalloc Fri Jan 18 13:15:25.033 [initandlisten] options: { dbpath: "/data/db/sconsTests/", nopreallocj: true, port: 27999, setParameter: [ "enableTestCommands=1" ] } Fri Jan 18 13:15:25.094 [initandlisten] journal dir=/data/db/sconsTests/journal Fri Jan 18 13:15:25.094 [initandlisten] recover : no journal files present, no recovery needed Fri Jan 18 13:15:25.145 [FileAllocator] allocating new datafile /data/db/sconsTests/local.ns, filling with zeroes... Fri Jan 18 13:15:25.145 [FileAllocator] creating directory /data/db/sconsTests/_tmp Fri Jan 18 13:15:25.479 [FileAllocator] done allocating datafile /data/db/sconsTests/local.ns, size: 16MB, took 0.293 secs Fri Jan 18 13:15:25.480 [FileAllocator] allocating new datafile /data/db/sconsTests/local.0, filling with zeroes... Fri Jan 18 13:15:26.011 [FileAllocator] done allocating datafile /data/db/sconsTests/local.0, size: 64MB, took 0.53 secs Fri Jan 18 13:15:26.012 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:37 867ms Fri Jan 18 13:15:26.012 [initandlisten] waiting for connections on port 27999 Fri Jan 18 13:15:26.012 [websvr] admin web console waiting for connections on port 28999 running /home/gregorv/Workspaces/10Gen/mongo/mongod --port 27999 --dbpath /data/db/sconsTests/ --setParameter enableTestCommands=1 --nopreallocj ******************************************* Test : sync_fast_switch.js ... Command : /home/gregorv/Workspaces/10Gen/mongo/mongo --port 27999 --authenticationMechanism MONGO-CR /home/gregorv/Workspaces/10Gen/mongo/jstests/replsets/sync_fast_switch.js --eval TestData = new Object();TestData.testPath = "/home/gregorv/Workspaces/10Gen/mongo/jstests/replsets/sync_fast_switch.js";TestData.testFile = "sync_fast_switch.js";TestData.testName = "sync_fast_switch";TestData.noJournal = false;TestData.noJournalPrealloc = true;TestData.auth = false;TestData.keyFile = null;TestData.keyFileData = null; Date : Fri Jan 18 13:15:27 2013 MongoDB shell version: 2.3.3-pre- connecting to: 127.0.0.1:27999/test Fri Jan 18 13:15:27.042 [initandlisten] connection accepted from 127.0.0.1:38954 #2 (1 connection now open) null Running: sudo bash -c tc qdisc del root dev lo > /tmp/output Fri Jan 18 13:15:27.046 shell: started program sudo bash -c tc qdisc del root dev lo > /tmp/output Running: sudo bash -c tc qdisc add dev lo handle 1: root htb > /tmp/output Fri Jan 18 13:15:27.061 shell: started program sudo bash -c tc qdisc add dev lo handle 1: root htb > /tmp/output Running: sudo bash -c tc class add dev lo parent 1: classid 1:1 htb rate 1000Mbps > /tmp/output Fri Jan 18 13:15:27.075 shell: started program sudo bash -c tc class add dev lo parent 1: classid 1:1 htb rate 1000Mbps > /tmp/output Running: sudo bash -c tc class add dev lo parent 1:1 classid 1:10 htb rate 1bps ceil 1000Mbps > /tmp/output Fri Jan 18 13:15:27.089 shell: started program sudo bash -c tc class add dev lo parent 1:1 classid 1:10 htb rate 1bps ceil 1000Mbps > /tmp/output ReplSetTest Starting Set ReplSetTest n is : 0 ReplSetTest n: 0 ports: [ 31000, 31001, 31002, 31003, 31004 ] 31000 number { "useHostName" : true, "oplogSize" : 40, "keyFile" : undefined, "port" : 31000, "noprealloc" : "", "smallfiles" : "", "rest" : "", "replSet" : "testReplSet", "dbpath" : "$set-$node", "useIP" : true, "restart" : undefined, "pathOpts" : { "node" : 0, "set" : "testReplSet" } } ReplSetTest Starting.... Using Local IP for host: 127.0.0.2 Resetting db path '/data/db/testReplSet-0' Fri Jan 18 13:15:27.164 shell: started program /home/gregorv/Workspaces/10Gen/mongo/mongod --oplogSize 40 --port 31000 --noprealloc --smallfiles --rest --replSet testReplSet --dbpath /data/db/testReplSet-0 --nopreallocj --bind_ip 127.0.0.2 --setParameter enableTestCommands=1 m31000| note: noprealloc may hurt performance in many applications m31000| Fri Jan 18 13:15:27.189 [initandlisten] MongoDB starting : pid=23848 port=31000 dbpath=/data/db/testReplSet-0 64-bit host=nuwen m31000| Fri Jan 18 13:15:27.189 [initandlisten] m31000| Fri Jan 18 13:15:27.189 [initandlisten] ** NOTE: This is a development version (2.3.3-pre-) of MongoDB. m31000| Fri Jan 18 13:15:27.189 [initandlisten] ** Not recommended for production. m31000| Fri Jan 18 13:15:27.189 [initandlisten] m31000| Fri Jan 18 13:15:27.189 [initandlisten] db version v2.3.3-pre-, pdfile version 4.5 m31000| Fri Jan 18 13:15:27.189 [initandlisten] git version: d3e8692ad5981bba5eaa50966ba0baf3147e7d9f m31000| Fri Jan 18 13:15:27.189 [initandlisten] build info: Linux nuwen 3.5.0-21-generic #32-Ubuntu SMP Tue Dec 11 18:51:59 UTC 2012 x86_64 BOOST_LIB_VERSION=1_49 m31000| Fri Jan 18 13:15:27.189 [initandlisten] allocator: tcmalloc m31000| Fri Jan 18 13:15:27.189 [initandlisten] options: { bind_ip: "127.0.0.2", dbpath: "/data/db/testReplSet-0", noprealloc: true, nopreallocj: true, oplogSize: 40, port: 31000, replSet: "testReplSet", rest: true, setParameter: [ "enableTestCommands=1" ], smallfiles: true } m31000| Fri Jan 18 13:15:27.270 [initandlisten] journal dir=/data/db/testReplSet-0/journal m31000| Fri Jan 18 13:15:27.270 [initandlisten] recover : no journal files present, no recovery needed m31000| Fri Jan 18 13:15:27.312 [FileAllocator] allocating new datafile /data/db/testReplSet-0/local.ns, filling with zeroes... m31000| Fri Jan 18 13:15:27.312 [FileAllocator] creating directory /data/db/testReplSet-0/_tmp m31000| Fri Jan 18 13:15:27.503 [FileAllocator] done allocating datafile /data/db/testReplSet-0/local.ns, size: 16MB, took 0.15 secs m31000| Fri Jan 18 13:15:27.504 [FileAllocator] allocating new datafile /data/db/testReplSet-0/local.0, filling with zeroes... m31000| Fri Jan 18 13:15:27.645 [FileAllocator] done allocating datafile /data/db/testReplSet-0/local.0, size: 16MB, took 0.141 secs m31000| Fri Jan 18 13:15:27.646 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:37 334ms m31000| Fri Jan 18 13:15:27.647 [initandlisten] waiting for connections on port 31000 m31000| Fri Jan 18 13:15:27.648 [websvr] admin web console waiting for connections on port 32000 m31000| Fri Jan 18 13:15:27.650 [rsStart] replSet can't get local.system.replset config from self or any seed (EMPTYCONFIG) m31000| Fri Jan 18 13:15:27.650 [rsStart] replSet info you may need to run replSetInitiate -- rs.initiate() in the shell -- if that is not already done m31000| Fri Jan 18 13:15:27.766 [initandlisten] connection accepted from 127.0.0.1:38664 #1 (1 connection now open) [ connection to 127.0.0.2:31000 ] ReplSetTest n is : 1 ReplSetTest n: 1 ports: [ 31000, 31001, 31002, 31003, 31004 ] 31001 number { "useHostName" : true, "oplogSize" : 40, "keyFile" : undefined, "port" : 31001, "noprealloc" : "", "smallfiles" : "", "rest" : "", "replSet" : "testReplSet", "dbpath" : "$set-$node", "useIP" : true, "restart" : undefined, "pathOpts" : { "node" : 1, "set" : "testReplSet" } } ReplSetTest Starting.... Using Local IP for host: 127.0.0.3 Resetting db path '/data/db/testReplSet-1' Fri Jan 18 13:15:27.821 shell: started program /home/gregorv/Workspaces/10Gen/mongo/mongod --oplogSize 40 --port 31001 --noprealloc --smallfiles --rest --replSet testReplSet --dbpath /data/db/testReplSet-1 --nopreallocj --bind_ip 127.0.0.3 --setParameter enableTestCommands=1 m31001| note: noprealloc may hurt performance in many applications m31001| Fri Jan 18 13:15:27.848 [initandlisten] MongoDB starting : pid=23899 port=31001 dbpath=/data/db/testReplSet-1 64-bit host=nuwen m31001| Fri Jan 18 13:15:27.848 [initandlisten] m31001| Fri Jan 18 13:15:27.848 [initandlisten] ** NOTE: This is a development version (2.3.3-pre-) of MongoDB. m31001| Fri Jan 18 13:15:27.848 [initandlisten] ** Not recommended for production. m31001| Fri Jan 18 13:15:27.848 [initandlisten] m31001| Fri Jan 18 13:15:27.848 [initandlisten] db version v2.3.3-pre-, pdfile version 4.5 m31001| Fri Jan 18 13:15:27.848 [initandlisten] git version: d3e8692ad5981bba5eaa50966ba0baf3147e7d9f m31001| Fri Jan 18 13:15:27.848 [initandlisten] build info: Linux nuwen 3.5.0-21-generic #32-Ubuntu SMP Tue Dec 11 18:51:59 UTC 2012 x86_64 BOOST_LIB_VERSION=1_49 m31001| Fri Jan 18 13:15:27.848 [initandlisten] allocator: tcmalloc m31001| Fri Jan 18 13:15:27.848 [initandlisten] options: { bind_ip: "127.0.0.3", dbpath: "/data/db/testReplSet-1", noprealloc: true, nopreallocj: true, oplogSize: 40, port: 31001, replSet: "testReplSet", rest: true, setParameter: [ "enableTestCommands=1" ], smallfiles: true } m31001| Fri Jan 18 13:15:27.904 [initandlisten] journal dir=/data/db/testReplSet-1/journal m31001| Fri Jan 18 13:15:27.905 [initandlisten] recover : no journal files present, no recovery needed m31001| Fri Jan 18 13:15:27.947 [FileAllocator] allocating new datafile /data/db/testReplSet-1/local.ns, filling with zeroes... m31001| Fri Jan 18 13:15:27.947 [FileAllocator] creating directory /data/db/testReplSet-1/_tmp m31001| Fri Jan 18 13:15:28.132 [FileAllocator] done allocating datafile /data/db/testReplSet-1/local.ns, size: 16MB, took 0.152 secs m31001| Fri Jan 18 13:15:28.133 [FileAllocator] allocating new datafile /data/db/testReplSet-1/local.0, filling with zeroes... m31001| Fri Jan 18 13:15:28.282 [FileAllocator] done allocating datafile /data/db/testReplSet-1/local.0, size: 16MB, took 0.149 secs m31001| Fri Jan 18 13:15:28.283 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:37 336ms m31001| Fri Jan 18 13:15:28.284 [initandlisten] waiting for connections on port 31001 m31001| Fri Jan 18 13:15:28.284 [websvr] admin web console waiting for connections on port 32001 m31001| Fri Jan 18 13:15:28.285 [rsStart] replSet can't get local.system.replset config from self or any seed (EMPTYCONFIG) m31001| Fri Jan 18 13:15:28.285 [rsStart] replSet info you may need to run replSetInitiate -- rs.initiate() in the shell -- if that is not already done m31001| Fri Jan 18 13:15:28.422 [initandlisten] connection accepted from 127.0.0.1:41784 #1 (1 connection now open) [ connection to 127.0.0.2:31000, connection to 127.0.0.3:31001 ] ReplSetTest n is : 2 ReplSetTest n: 2 ports: [ 31000, 31001, 31002, 31003, 31004 ] 31002 number { "useHostName" : true, "oplogSize" : 40, "keyFile" : undefined, "port" : 31002, "noprealloc" : "", "smallfiles" : "", "rest" : "", "replSet" : "testReplSet", "dbpath" : "$set-$node", "useIP" : true, "restart" : undefined, "pathOpts" : { "node" : 2, "set" : "testReplSet" } } ReplSetTest Starting.... Using Local IP for host: 127.0.0.4 Resetting db path '/data/db/testReplSet-2' Fri Jan 18 13:15:28.479 shell: started program /home/gregorv/Workspaces/10Gen/mongo/mongod --oplogSize 40 --port 31002 --noprealloc --smallfiles --rest --replSet testReplSet --dbpath /data/db/testReplSet-2 --nopreallocj --bind_ip 127.0.0.4 --setParameter enableTestCommands=1 m31002| note: noprealloc may hurt performance in many applications m31002| Fri Jan 18 13:15:28.502 [initandlisten] MongoDB starting : pid=23951 port=31002 dbpath=/data/db/testReplSet-2 64-bit host=nuwen m31002| Fri Jan 18 13:15:28.502 [initandlisten] m31002| Fri Jan 18 13:15:28.502 [initandlisten] ** NOTE: This is a development version (2.3.3-pre-) of MongoDB. m31002| Fri Jan 18 13:15:28.502 [initandlisten] ** Not recommended for production. m31002| Fri Jan 18 13:15:28.502 [initandlisten] m31002| Fri Jan 18 13:15:28.502 [initandlisten] db version v2.3.3-pre-, pdfile version 4.5 m31002| Fri Jan 18 13:15:28.502 [initandlisten] git version: d3e8692ad5981bba5eaa50966ba0baf3147e7d9f m31002| Fri Jan 18 13:15:28.502 [initandlisten] build info: Linux nuwen 3.5.0-21-generic #32-Ubuntu SMP Tue Dec 11 18:51:59 UTC 2012 x86_64 BOOST_LIB_VERSION=1_49 m31002| Fri Jan 18 13:15:28.502 [initandlisten] allocator: tcmalloc m31002| Fri Jan 18 13:15:28.502 [initandlisten] options: { bind_ip: "127.0.0.4", dbpath: "/data/db/testReplSet-2", noprealloc: true, nopreallocj: true, oplogSize: 40, port: 31002, replSet: "testReplSet", rest: true, setParameter: [ "enableTestCommands=1" ], smallfiles: true } m31002| Fri Jan 18 13:15:28.567 [initandlisten] journal dir=/data/db/testReplSet-2/journal m31002| Fri Jan 18 13:15:28.567 [initandlisten] recover : no journal files present, no recovery needed m31002| Fri Jan 18 13:15:28.617 [FileAllocator] allocating new datafile /data/db/testReplSet-2/local.ns, filling with zeroes... m31002| Fri Jan 18 13:15:28.618 [FileAllocator] creating directory /data/db/testReplSet-2/_tmp m31002| Fri Jan 18 13:15:28.801 [FileAllocator] done allocating datafile /data/db/testReplSet-2/local.ns, size: 16MB, took 0.15 secs m31002| Fri Jan 18 13:15:28.801 [FileAllocator] allocating new datafile /data/db/testReplSet-2/local.0, filling with zeroes... m31002| Fri Jan 18 13:15:28.942 [FileAllocator] done allocating datafile /data/db/testReplSet-2/local.0, size: 16MB, took 0.141 secs m31002| Fri Jan 18 13:15:28.943 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:37 326ms m31002| Fri Jan 18 13:15:28.944 [initandlisten] waiting for connections on port 31002 m31002| Fri Jan 18 13:15:28.944 [websvr] admin web console waiting for connections on port 32002 m31002| Fri Jan 18 13:15:28.946 [rsStart] replSet can't get local.system.replset config from self or any seed (EMPTYCONFIG) m31002| Fri Jan 18 13:15:28.946 [rsStart] replSet info you may need to run replSetInitiate -- rs.initiate() in the shell -- if that is not already done m31002| Fri Jan 18 13:15:29.080 [initandlisten] connection accepted from 127.0.0.1:42684 #1 (1 connection now open) [ connection to 127.0.0.2:31000, connection to 127.0.0.3:31001, connection to 127.0.0.4:31002 ] ReplSetTest n is : 3 ReplSetTest n: 3 ports: [ 31000, 31001, 31002, 31003, 31004 ] 31003 number { "useHostName" : true, "oplogSize" : 40, "keyFile" : undefined, "port" : 31003, "noprealloc" : "", "smallfiles" : "", "rest" : "", "replSet" : "testReplSet", "dbpath" : "$set-$node", "useIP" : true, "restart" : undefined, "pathOpts" : { "node" : 3, "set" : "testReplSet" } } ReplSetTest Starting.... Using Local IP for host: 127.0.0.5 Resetting db path '/data/db/testReplSet-3' Fri Jan 18 13:15:29.118 shell: started program /home/gregorv/Workspaces/10Gen/mongo/mongod --oplogSize 40 --port 31003 --noprealloc --smallfiles --rest --replSet testReplSet --dbpath /data/db/testReplSet-3 --nopreallocj --bind_ip 127.0.0.5 --setParameter enableTestCommands=1 m31003| note: noprealloc may hurt performance in many applications m31003| Fri Jan 18 13:15:29.145 [initandlisten] MongoDB starting : pid=24002 port=31003 dbpath=/data/db/testReplSet-3 64-bit host=nuwen m31003| Fri Jan 18 13:15:29.145 [initandlisten] m31003| Fri Jan 18 13:15:29.145 [initandlisten] ** NOTE: This is a development version (2.3.3-pre-) of MongoDB. m31003| Fri Jan 18 13:15:29.145 [initandlisten] ** Not recommended for production. m31003| Fri Jan 18 13:15:29.145 [initandlisten] m31003| Fri Jan 18 13:15:29.145 [initandlisten] db version v2.3.3-pre-, pdfile version 4.5 m31003| Fri Jan 18 13:15:29.145 [initandlisten] git version: d3e8692ad5981bba5eaa50966ba0baf3147e7d9f m31003| Fri Jan 18 13:15:29.145 [initandlisten] build info: Linux nuwen 3.5.0-21-generic #32-Ubuntu SMP Tue Dec 11 18:51:59 UTC 2012 x86_64 BOOST_LIB_VERSION=1_49 m31003| Fri Jan 18 13:15:29.145 [initandlisten] allocator: tcmalloc m31003| Fri Jan 18 13:15:29.145 [initandlisten] options: { bind_ip: "127.0.0.5", dbpath: "/data/db/testReplSet-3", noprealloc: true, nopreallocj: true, oplogSize: 40, port: 31003, replSet: "testReplSet", rest: true, setParameter: [ "enableTestCommands=1" ], smallfiles: true } m31003| Fri Jan 18 13:15:29.201 [initandlisten] journal dir=/data/db/testReplSet-3/journal m31003| Fri Jan 18 13:15:29.202 [initandlisten] recover : no journal files present, no recovery needed m31003| Fri Jan 18 13:15:29.252 [FileAllocator] allocating new datafile /data/db/testReplSet-3/local.ns, filling with zeroes... m31003| Fri Jan 18 13:15:29.252 [FileAllocator] creating directory /data/db/testReplSet-3/_tmp m31003| Fri Jan 18 13:15:29.435 [FileAllocator] done allocating datafile /data/db/testReplSet-3/local.ns, size: 16MB, took 0.15 secs m31003| Fri Jan 18 13:15:29.436 [FileAllocator] allocating new datafile /data/db/testReplSet-3/local.0, filling with zeroes... m31003| Fri Jan 18 13:15:29.577 [FileAllocator] done allocating datafile /data/db/testReplSet-3/local.0, size: 16MB, took 0.141 secs m31003| Fri Jan 18 13:15:29.578 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:37 325ms m31003| Fri Jan 18 13:15:29.578 [initandlisten] waiting for connections on port 31003 m31003| Fri Jan 18 13:15:29.579 [websvr] admin web console waiting for connections on port 32003 m31003| Fri Jan 18 13:15:29.581 [rsStart] replSet can't get local.system.replset config from self or any seed (EMPTYCONFIG) m31003| Fri Jan 18 13:15:29.581 [rsStart] replSet info you may need to run replSetInitiate -- rs.initiate() in the shell -- if that is not already done m31003| Fri Jan 18 13:15:29.720 [initandlisten] connection accepted from 127.0.0.1:60105 #1 (1 connection now open) [ connection to 127.0.0.2:31000, connection to 127.0.0.3:31001, connection to 127.0.0.4:31002, connection to 127.0.0.5:31003 ] ReplSetTest n is : 4 ReplSetTest n: 4 ports: [ 31000, 31001, 31002, 31003, 31004 ] 31004 number { "useHostName" : true, "oplogSize" : 40, "keyFile" : undefined, "port" : 31004, "noprealloc" : "", "smallfiles" : "", "rest" : "", "replSet" : "testReplSet", "dbpath" : "$set-$node", "useIP" : true, "restart" : undefined, "pathOpts" : { "node" : 4, "set" : "testReplSet" } } ReplSetTest Starting.... Using Local IP for host: 127.0.0.6 Resetting db path '/data/db/testReplSet-4' Fri Jan 18 13:15:29.777 shell: started program /home/gregorv/Workspaces/10Gen/mongo/mongod --oplogSize 40 --port 31004 --noprealloc --smallfiles --rest --replSet testReplSet --dbpath /data/db/testReplSet-4 --nopreallocj --bind_ip 127.0.0.6 --setParameter enableTestCommands=1 m31004| note: noprealloc may hurt performance in many applications m31004| Fri Jan 18 13:15:29.804 [initandlisten] MongoDB starting : pid=24053 port=31004 dbpath=/data/db/testReplSet-4 64-bit host=nuwen m31004| Fri Jan 18 13:15:29.804 [initandlisten] m31004| Fri Jan 18 13:15:29.804 [initandlisten] ** NOTE: This is a development version (2.3.3-pre-) of MongoDB. m31004| Fri Jan 18 13:15:29.804 [initandlisten] ** Not recommended for production. m31004| Fri Jan 18 13:15:29.804 [initandlisten] m31004| Fri Jan 18 13:15:29.804 [initandlisten] db version v2.3.3-pre-, pdfile version 4.5 m31004| Fri Jan 18 13:15:29.804 [initandlisten] git version: d3e8692ad5981bba5eaa50966ba0baf3147e7d9f m31004| Fri Jan 18 13:15:29.804 [initandlisten] build info: Linux nuwen 3.5.0-21-generic #32-Ubuntu SMP Tue Dec 11 18:51:59 UTC 2012 x86_64 BOOST_LIB_VERSION=1_49 m31004| Fri Jan 18 13:15:29.804 [initandlisten] allocator: tcmalloc m31004| Fri Jan 18 13:15:29.804 [initandlisten] options: { bind_ip: "127.0.0.6", dbpath: "/data/db/testReplSet-4", noprealloc: true, nopreallocj: true, oplogSize: 40, port: 31004, replSet: "testReplSet", rest: true, setParameter: [ "enableTestCommands=1" ], smallfiles: true } m31004| Fri Jan 18 13:15:29.870 [initandlisten] journal dir=/data/db/testReplSet-4/journal m31004| Fri Jan 18 13:15:29.870 [initandlisten] recover : no journal files present, no recovery needed m31004| Fri Jan 18 13:15:29.912 [FileAllocator] allocating new datafile /data/db/testReplSet-4/local.ns, filling with zeroes... m31004| Fri Jan 18 13:15:29.912 [FileAllocator] creating directory /data/db/testReplSet-4/_tmp m31004| Fri Jan 18 13:15:30.095 [FileAllocator] done allocating datafile /data/db/testReplSet-4/local.ns, size: 16MB, took 0.15 secs m31004| Fri Jan 18 13:15:30.096 [FileAllocator] allocating new datafile /data/db/testReplSet-4/local.0, filling with zeroes... m31004| Fri Jan 18 13:15:30.237 [FileAllocator] done allocating datafile /data/db/testReplSet-4/local.0, size: 16MB, took 0.141 secs m31004| Fri Jan 18 13:15:30.238 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:37 326ms m31004| Fri Jan 18 13:15:30.238 [initandlisten] waiting for connections on port 31004 m31004| Fri Jan 18 13:15:30.239 [websvr] admin web console waiting for connections on port 32004 m31004| Fri Jan 18 13:15:30.240 [rsStart] replSet can't get local.system.replset config from self or any seed (EMPTYCONFIG) m31004| Fri Jan 18 13:15:30.240 [rsStart] replSet info you may need to run replSetInitiate -- rs.initiate() in the shell -- if that is not already done m31004| Fri Jan 18 13:15:30.379 [initandlisten] connection accepted from 127.0.0.1:59873 #1 (1 connection now open) [ connection to 127.0.0.2:31000, connection to 127.0.0.3:31001, connection to 127.0.0.4:31002, connection to 127.0.0.5:31003, connection to 127.0.0.6:31004 ] { "replSetInitiate" : { "_id" : "testReplSet", "members" : [ { "_id" : 0, "host" : "127.0.0.2:31000" }, { "_id" : 1, "host" : "127.0.0.3:31001" }, { "_id" : 2, "host" : "127.0.0.4:31002" }, { "_id" : 3, "host" : "127.0.0.5:31003" }, { "_id" : 4, "host" : "127.0.0.6:31004" } ] } } m31000| Fri Jan 18 13:15:30.381 [conn1] replSet replSetInitiate admin command received from client m31000| Fri Jan 18 13:15:30.382 [conn1] replSet replSetInitiate config object parses ok, 5 members specified m31001| Fri Jan 18 13:15:30.382 [initandlisten] connection accepted from 127.0.0.2:34845 #2 (2 connections now open) m31002| Fri Jan 18 13:15:30.383 [initandlisten] connection accepted from 127.0.0.2:35885 #2 (2 connections now open) m31003| Fri Jan 18 13:15:30.384 [initandlisten] connection accepted from 127.0.0.2:55010 #2 (2 connections now open) m31004| Fri Jan 18 13:15:30.384 [initandlisten] connection accepted from 127.0.0.2:55988 #2 (2 connections now open) m31000| Fri Jan 18 13:15:30.385 [conn1] replSet replSetInitiate all members seem up m31000| Fri Jan 18 13:15:30.385 [conn1] ****** m31000| Fri Jan 18 13:15:30.385 [conn1] creating replication oplog of size: 40MB... m31000| Fri Jan 18 13:15:30.385 [FileAllocator] allocating new datafile /data/db/testReplSet-0/local.1, filling with zeroes... m31000| Fri Jan 18 13:15:30.912 [FileAllocator] done allocating datafile /data/db/testReplSet-0/local.1, size: 64MB, took 0.526 secs m31000| Fri Jan 18 13:15:32.003 [conn1] ****** m31000| Fri Jan 18 13:15:32.003 [conn1] replSet info saving a newer config version to local.system.replset m31000| Fri Jan 18 13:15:32.015 [conn1] replSet saveConfigLocally done m31000| Fri Jan 18 13:15:32.015 [conn1] replSet replSetInitiate config now saved locally. Should come online in about a minute. m31000| Fri Jan 18 13:15:32.016 [conn1] build index local.replset.minvalid { _id: 1 } m31000| Fri Jan 18 13:15:32.016 [conn1] build index done. scanned 0 total records. 0 secs m31000| Fri Jan 18 13:15:32.016 [conn1] command admin.$cmd command: { replSetInitiate: { _id: "testReplSet", members: [ { _id: 0.0, host: "127.0.0.2:31000" }, { _id: 1.0, host: "127.0.0.3:31001" }, { _id: 2.0, host: "127.0.0.4:31002" }, { _id: 3.0, host: "127.0.0.5:31003" }, { _id: 4.0, host: "127.0.0.6:31004" } ] } } ntoreturn:1 keyUpdates:0 locks(micros) W:1631436 reslen:112 1634ms { "info" : "Config now saved locally. Should come online in about a minute.", "ok" : 1 } m31000| Fri Jan 18 13:15:37.650 [rsStart] replSet I am 127.0.0.2:31000 m31000| Fri Jan 18 13:15:37.650 [rsStart] replSet STARTUP2 m31000| Fri Jan 18 13:15:37.651 [rsHealthPoll] replSet member 127.0.0.3:31001 is up m31000| Fri Jan 18 13:15:37.651 [rsHealthPoll] replSet member 127.0.0.6:31004 is up m31000| Fri Jan 18 13:15:37.651 [rsHealthPoll] replSet member 127.0.0.5:31003 is up m31002| Fri Jan 18 13:15:37.651 [conn2] end connection 127.0.0.2:35885 (1 connection now open) m31002| Fri Jan 18 13:15:37.651 [initandlisten] connection accepted from 127.0.0.2:44093 #3 (3 connections now open) m31000| Fri Jan 18 13:15:37.651 [rsHealthPoll] replSet member 127.0.0.4:31002 is up m31001| Fri Jan 18 13:15:38.286 [rsStart] trying to contact 127.0.0.2:31000 m31000| Fri Jan 18 13:15:38.286 [initandlisten] connection accepted from 127.0.0.3:38111 #2 (2 connections now open) m31001| Fri Jan 18 13:15:38.287 [rsStart] replSet I am 127.0.0.3:31001 m31001| Fri Jan 18 13:15:38.287 [rsStart] replSet got config version 1 from a remote, saving locally m31001| Fri Jan 18 13:15:38.287 [rsStart] replSet info saving a newer config version to local.system.replset m31001| Fri Jan 18 13:15:38.500 [rsStart] replSet saveConfigLocally done m31001| Fri Jan 18 13:15:38.500 [rsStart] replSet STARTUP2 m31001| Fri Jan 18 13:15:38.501 [rsSync] ****** m31001| Fri Jan 18 13:15:38.501 [rsSync] creating replication oplog of size: 40MB... m31001| Fri Jan 18 13:15:38.534 [FileAllocator] allocating new datafile /data/db/testReplSet-1/local.1, filling with zeroes... m31000| Fri Jan 18 13:15:38.651 [rsSync] replSet SECONDARY m31002| Fri Jan 18 13:15:38.946 [rsStart] trying to contact 127.0.0.2:31000 m31000| Fri Jan 18 13:15:38.947 [initandlisten] connection accepted from 127.0.0.4:48144 #3 (3 connections now open) m31002| Fri Jan 18 13:15:38.947 [rsStart] replSet I am 127.0.0.4:31002 m31002| Fri Jan 18 13:15:38.947 [rsStart] replSet got config version 1 from a remote, saving locally m31002| Fri Jan 18 13:15:38.947 [rsStart] replSet info saving a newer config version to local.system.replset m31001| Fri Jan 18 13:15:39.158 [FileAllocator] done allocating datafile /data/db/testReplSet-1/local.1, size: 64MB, took 0.623 secs m31002| Fri Jan 18 13:15:39.159 [rsStart] replSet saveConfigLocally done m31002| Fri Jan 18 13:15:39.159 [rsStart] replSet STARTUP2 m31002| Fri Jan 18 13:15:39.159 [rsSync] ****** m31002| Fri Jan 18 13:15:39.159 [rsSync] creating replication oplog of size: 40MB... m31002| Fri Jan 18 13:15:39.384 [FileAllocator] allocating new datafile /data/db/testReplSet-2/local.1, filling with zeroes... m31003| Fri Jan 18 13:15:39.581 [rsStart] trying to contact 127.0.0.2:31000 m31000| Fri Jan 18 13:15:39.581 [initandlisten] connection accepted from 127.0.0.5:40894 #4 (4 connections now open) m31003| Fri Jan 18 13:15:39.582 [rsStart] replSet I am 127.0.0.5:31003 m31003| Fri Jan 18 13:15:39.582 [rsStart] replSet got config version 1 from a remote, saving locally m31003| Fri Jan 18 13:15:39.582 [rsStart] replSet info saving a newer config version to local.system.replset m31000| Fri Jan 18 13:15:39.651 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Fri Jan 18 13:15:39.651 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state STARTUP2 m31000| Fri Jan 18 13:15:39.651 [rsMgr] not electing self, 127.0.0.4:31002 would veto with 'I don't think 127.0.0.2:31000 is electable' m31000| Fri Jan 18 13:15:39.652 [rsHealthPoll] replset info 127.0.0.4:31002 thinks that we are down m31000| Fri Jan 18 13:15:39.652 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state STARTUP2 m31000| Fri Jan 18 13:15:39.652 [rsMgr] not electing self, 127.0.0.4:31002 would veto with 'I don't think 127.0.0.2:31000 is electable' m31001| Fri Jan 18 13:15:40.199 [rsSync] ****** m31001| Fri Jan 18 13:15:40.199 [rsSync] replSet initial sync pending m31001| Fri Jan 18 13:15:40.199 [rsSync] replSet initial sync need a member to be primary or secondary to do our initial sync m31002| Fri Jan 18 13:15:40.199 [FileAllocator] done allocating datafile /data/db/testReplSet-2/local.1, size: 64MB, took 0.815 secs m31003| Fri Jan 18 13:15:40.235 [rsStart] replSet saveConfigLocally done m31003| Fri Jan 18 13:15:40.236 [rsStart] replSet STARTUP2 m31003| Fri Jan 18 13:15:40.236 [rsSync] ****** m31003| Fri Jan 18 13:15:40.236 [rsSync] creating replication oplog of size: 40MB... m31004| Fri Jan 18 13:15:40.241 [rsStart] trying to contact 127.0.0.2:31000 m31000| Fri Jan 18 13:15:40.241 [initandlisten] connection accepted from 127.0.0.6:46416 #5 (5 connections now open) m31004| Fri Jan 18 13:15:40.242 [rsStart] replSet I am 127.0.0.6:31004 m31004| Fri Jan 18 13:15:40.242 [rsStart] replSet got config version 1 from a remote, saving locally m31004| Fri Jan 18 13:15:40.242 [rsStart] replSet info saving a newer config version to local.system.replset m31004| Fri Jan 18 13:15:40.287 [initandlisten] connection accepted from 127.0.0.3:35741 #3 (3 connections now open) m31003| Fri Jan 18 13:15:40.287 [initandlisten] connection accepted from 127.0.0.3:55617 #3 (3 connections now open) m31001| Fri Jan 18 13:15:40.288 [rsHealthPoll] replSet member 127.0.0.2:31000 is up m31001| Fri Jan 18 13:15:40.288 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31002| Fri Jan 18 13:15:40.288 [initandlisten] connection accepted from 127.0.0.3:50234 #4 (3 connections now open) m31001| Fri Jan 18 13:15:40.288 [rsHealthPoll] replSet member 127.0.0.6:31004 is up m31001| Fri Jan 18 13:15:40.288 [rsHealthPoll] replset info 127.0.0.5:31003 thinks that we are down m31001| Fri Jan 18 13:15:40.288 [rsHealthPoll] replSet member 127.0.0.5:31003 is up m31001| Fri Jan 18 13:15:40.288 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state STARTUP2 m31002| Fri Jan 18 13:15:40.288 [conn4] end connection 127.0.0.3:50234 (2 connections now open) m31002| Fri Jan 18 13:15:40.288 [initandlisten] connection accepted from 127.0.0.3:41067 #5 (4 connections now open) m31001| Fri Jan 18 13:15:40.288 [rsHealthPoll] replset info 127.0.0.4:31002 thinks that we are down m31001| Fri Jan 18 13:15:40.288 [rsHealthPoll] replSet member 127.0.0.4:31002 is up m31001| Fri Jan 18 13:15:40.288 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state STARTUP2 m31003| Fri Jan 18 13:15:40.658 [FileAllocator] allocating new datafile /data/db/testReplSet-3/local.1, filling with zeroes... m31004| Fri Jan 18 13:15:40.658 [rsStart] replSet saveConfigLocally done m31004| Fri Jan 18 13:15:40.659 [rsStart] replSet STARTUP2 m31004| Fri Jan 18 13:15:40.659 [rsSync] ****** m31004| Fri Jan 18 13:15:40.659 [rsSync] creating replication oplog of size: 40MB... m31002| Fri Jan 18 13:15:40.947 [rsHealthPoll] replSet member 127.0.0.2:31000 is up m31003| Fri Jan 18 13:15:40.948 [initandlisten] connection accepted from 127.0.0.4:44908 #4 (4 connections now open) m31002| Fri Jan 18 13:15:40.948 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31004| Fri Jan 18 13:15:40.948 [initandlisten] connection accepted from 127.0.0.4:37410 #4 (4 connections now open) m31001| Fri Jan 18 13:15:40.948 [initandlisten] connection accepted from 127.0.0.4:42413 #3 (3 connections now open) m31002| Fri Jan 18 13:15:40.948 [rsHealthPoll] replset info 127.0.0.5:31003 thinks that we are down m31002| Fri Jan 18 13:15:40.948 [rsHealthPoll] replSet member 127.0.0.5:31003 is up m31002| Fri Jan 18 13:15:40.948 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state STARTUP2 m31001| Fri Jan 18 13:15:40.948 [initandlisten] connection accepted from 127.0.0.4:46516 #4 (4 connections now open) m31001| Fri Jan 18 13:15:40.948 [conn3] end connection 127.0.0.4:42413 (3 connections now open) m31002| Fri Jan 18 13:15:40.948 [rsHealthPoll] replset info 127.0.0.6:31004 thinks that we are down m31002| Fri Jan 18 13:15:40.948 [rsHealthPoll] replSet member 127.0.0.6:31004 is up m31002| Fri Jan 18 13:15:40.948 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state STARTUP2 m31002| Fri Jan 18 13:15:40.948 [rsHealthPoll] replSet member 127.0.0.3:31001 is up m31002| Fri Jan 18 13:15:40.948 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state STARTUP2 m31004| Fri Jan 18 13:15:41.358 [FileAllocator] allocating new datafile /data/db/testReplSet-4/local.1, filling with zeroes... m31002| Fri Jan 18 13:15:41.416 [rsSync] ****** m31002| Fri Jan 18 13:15:41.416 [rsSync] replSet initial sync pending m31002| Fri Jan 18 13:15:41.416 [rsSync] replSet initial sync need a member to be primary or secondary to do our initial sync m31003| Fri Jan 18 13:15:41.416 [FileAllocator] done allocating datafile /data/db/testReplSet-3/local.1, size: 64MB, took 0.758 secs m31004| Fri Jan 18 13:15:41.582 [initandlisten] connection accepted from 127.0.0.5:52083 #5 (5 connections now open) m31002| Fri Jan 18 13:15:41.582 [initandlisten] connection accepted from 127.0.0.5:51726 #6 (4 connections now open) m31001| Fri Jan 18 13:15:41.582 [initandlisten] connection accepted from 127.0.0.5:49428 #5 (4 connections now open) m31001| Fri Jan 18 13:15:41.582 [initandlisten] connection accepted from 127.0.0.5:52703 #6 (5 connections now open) m31003| Fri Jan 18 13:15:41.582 [rsHealthPoll] replSet member 127.0.0.2:31000 is up m31003| Fri Jan 18 13:15:41.582 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31001| Fri Jan 18 13:15:41.582 [conn5] end connection 127.0.0.5:49428 (4 connections now open) m31003| Fri Jan 18 13:15:41.583 [rsHealthPoll] replSet member 127.0.0.4:31002 is up m31003| Fri Jan 18 13:15:41.583 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state STARTUP2 m31003| Fri Jan 18 13:15:41.583 [rsHealthPoll] replset info 127.0.0.6:31004 thinks that we are down m31003| Fri Jan 18 13:15:41.583 [rsHealthPoll] replSet member 127.0.0.6:31004 is up m31003| Fri Jan 18 13:15:41.583 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state STARTUP2 m31003| Fri Jan 18 13:15:41.583 [rsHealthPoll] replSet member 127.0.0.3:31001 is up m31003| Fri Jan 18 13:15:41.583 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state STARTUP2 m31000| Fri Jan 18 13:15:41.651 [rsHealthPoll] replset info 127.0.0.6:31004 thinks that we are down m31000| Fri Jan 18 13:15:41.651 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state STARTUP2 m31000| Fri Jan 18 13:15:41.651 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state STARTUP2 m31000| Fri Jan 18 13:15:41.652 [rsMgr] not electing self, 127.0.0.6:31004 would veto with 'I don't think 127.0.0.2:31000 is electable' m31000| Fri Jan 18 13:15:41.652 [rsMgr] not electing self, 127.0.0.6:31004 would veto with 'I don't think 127.0.0.2:31000 is electable' m31003| Fri Jan 18 13:15:42.243 [initandlisten] connection accepted from 127.0.0.6:42167 #5 (5 connections now open) m31004| Fri Jan 18 13:15:42.243 [rsHealthPoll] replSet member 127.0.0.2:31000 is up m31004| Fri Jan 18 13:15:42.243 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31001| Fri Jan 18 13:15:42.243 [initandlisten] connection accepted from 127.0.0.6:48970 #7 (5 connections now open) m31002| Fri Jan 18 13:15:42.243 [initandlisten] connection accepted from 127.0.0.6:41886 #7 (5 connections now open) m31001| Fri Jan 18 13:15:42.243 [conn7] end connection 127.0.0.6:48970 (4 connections now open) m31001| Fri Jan 18 13:15:42.243 [initandlisten] connection accepted from 127.0.0.6:32875 #8 (6 connections now open) m31004| Fri Jan 18 13:15:42.243 [rsHealthPoll] replSet member 127.0.0.5:31003 is up m31004| Fri Jan 18 13:15:42.243 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state STARTUP2 m31004| Fri Jan 18 13:15:42.243 [rsHealthPoll] replSet member 127.0.0.4:31002 is up m31004| Fri Jan 18 13:15:42.243 [rsHealthPoll] replSet member 127.0.0.3:31001 is up m31004| Fri Jan 18 13:15:42.243 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state STARTUP2 m31004| Fri Jan 18 13:15:42.243 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state STARTUP2 m31001| Fri Jan 18 13:15:42.288 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state STARTUP2 m31004| Fri Jan 18 13:15:42.300 [FileAllocator] done allocating datafile /data/db/testReplSet-4/local.1, size: 64MB, took 0.941 secs m31003| Fri Jan 18 13:15:42.300 [rsSync] ****** m31003| Fri Jan 18 13:15:42.300 [rsSync] replSet initial sync pending m31003| Fri Jan 18 13:15:42.300 [rsSync] replSet initial sync need a member to be primary or secondary to do our initial sync m31004| Fri Jan 18 13:15:43.208 [rsSync] ****** m31004| Fri Jan 18 13:15:43.208 [rsSync] replSet initial sync pending m31004| Fri Jan 18 13:15:43.208 [rsSync] replSet initial sync need a member to be primary or secondary to do our initial sync m31000| Fri Jan 18 13:15:47.653 [rsMgr] replSet info electSelf 0 m31002| Fri Jan 18 13:15:47.653 [conn3] replSet RECOVERING m31002| Fri Jan 18 13:15:47.653 [conn3] replSet info voting yea for 127.0.0.2:31000 (0) m31003| Fri Jan 18 13:15:47.653 [conn2] replSet RECOVERING m31003| Fri Jan 18 13:15:47.653 [conn2] replSet info voting yea for 127.0.0.2:31000 (0) m31004| Fri Jan 18 13:15:47.653 [conn2] replSet RECOVERING m31004| Fri Jan 18 13:15:47.653 [conn2] replSet info voting yea for 127.0.0.2:31000 (0) m31001| Fri Jan 18 13:15:47.653 [conn2] replSet RECOVERING m31001| Fri Jan 18 13:15:47.653 [conn2] replSet info voting yea for 127.0.0.2:31000 (0) m31004| Fri Jan 18 13:15:48.244 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state RECOVERING m31004| Fri Jan 18 13:15:48.244 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state RECOVERING m31004| Fri Jan 18 13:15:48.244 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state RECOVERING m31001| Fri Jan 18 13:15:48.289 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state RECOVERING m31001| Fri Jan 18 13:15:48.289 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state RECOVERING m31001| Fri Jan 18 13:15:48.289 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state RECOVERING m31000| Fri Jan 18 13:15:48.652 [rsMgr] replSet PRIMARY m31002| Fri Jan 18 13:15:48.949 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state PRIMARY m31002| Fri Jan 18 13:15:48.949 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state RECOVERING m31002| Fri Jan 18 13:15:48.949 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state RECOVERING m31002| Fri Jan 18 13:15:48.949 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state RECOVERING m31003| Fri Jan 18 13:15:49.584 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state PRIMARY m31003| Fri Jan 18 13:15:49.584 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state RECOVERING m31003| Fri Jan 18 13:15:49.584 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state RECOVERING m31003| Fri Jan 18 13:15:49.584 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state RECOVERING m31000| Fri Jan 18 13:15:49.653 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state RECOVERING m31000| Fri Jan 18 13:15:49.653 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state RECOVERING m31000| Fri Jan 18 13:15:49.653 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state RECOVERING m31000| Fri Jan 18 13:15:49.653 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state RECOVERING m31004| Fri Jan 18 13:15:50.244 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state PRIMARY m31001| Fri Jan 18 13:15:50.289 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state PRIMARY m31003| Fri Jan 18 13:15:51.653 [conn2] end connection 127.0.0.2:55010 (4 connections now open) m31003| Fri Jan 18 13:15:51.653 [initandlisten] connection accepted from 127.0.0.2:39162 #6 (5 connections now open) m31001| Fri Jan 18 13:15:53.653 [conn2] end connection 127.0.0.2:34845 (4 connections now open) m31001| Fri Jan 18 13:15:53.653 [initandlisten] connection accepted from 127.0.0.2:58231 #9 (5 connections now open) m31003| Fri Jan 18 13:15:54.290 [conn3] end connection 127.0.0.3:55617 (4 connections now open) m31003| Fri Jan 18 13:15:54.290 [initandlisten] connection accepted from 127.0.0.3:36517 #7 (5 connections now open) m31003| Fri Jan 18 13:15:54.950 [conn4] end connection 127.0.0.4:44908 (4 connections now open) m31003| Fri Jan 18 13:15:54.950 [initandlisten] connection accepted from 127.0.0.4:50741 #8 (6 connections now open) m31002| Fri Jan 18 13:15:55.585 [conn6] end connection 127.0.0.5:51726 (4 connections now open) m31002| Fri Jan 18 13:15:55.585 [initandlisten] connection accepted from 127.0.0.5:52675 #8 (6 connections now open) m31001| Fri Jan 18 13:15:56.200 [rsSync] replSet initial sync pending m31001| Fri Jan 18 13:15:56.200 [rsSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31001| Fri Jan 18 13:15:56.200 [rsSync] SYNCED TO 127.0.0.2:31000 m31001| Fri Jan 18 13:15:56.200 [rsSync] replSet syncing to: 127.0.0.2:31000 m31000| Fri Jan 18 13:15:56.200 [initandlisten] connection accepted from 127.0.0.3:38940 #6 (6 connections now open) m31001| Fri Jan 18 13:15:56.201 [rsSync] build index local.me { _id: 1 } m31001| Fri Jan 18 13:15:56.202 [rsSync] build index done. scanned 0 total records. 0 secs m31001| Fri Jan 18 13:15:56.203 [rsSync] replSet initial sync drop all databases m31001| Fri Jan 18 13:15:56.203 [rsSync] dropAllDatabasesExceptLocal 1 m31001| Fri Jan 18 13:15:56.203 [rsSync] replSet initial sync clone all databases m31001| Fri Jan 18 13:15:56.203 [rsSync] replSet initial sync data copy, starting syncup m31001| Fri Jan 18 13:15:56.203 [rsSync] oplog sync 1 of 3 m31001| Fri Jan 18 13:15:56.204 [rsSync] oplog sync 2 of 3 m31001| Fri Jan 18 13:15:56.204 [rsSync] replSet initial sync building indexes m31001| Fri Jan 18 13:15:56.204 [rsSync] oplog sync 3 of 3 m31001| Fri Jan 18 13:15:56.204 [rsSync] replSet initial sync finishing up m31001| Fri Jan 18 13:15:56.205 [rsSync] replSet set minValid=50f99144:1 m31001| Fri Jan 18 13:15:56.205 [rsSync] build index local.replset.minvalid { _id: 1 } m31001| Fri Jan 18 13:15:56.205 [rsSync] build index done. scanned 0 total records. 0 secs m31001| Fri Jan 18 13:15:56.206 [rsSync] replSet initial sync done m31000| Fri Jan 18 13:15:56.206 [conn6] end connection 127.0.0.3:38940 (5 connections now open) m31002| Fri Jan 18 13:15:56.245 [conn7] end connection 127.0.0.6:41886 (4 connections now open) m31002| Fri Jan 18 13:15:56.245 [initandlisten] connection accepted from 127.0.0.6:54288 #9 (5 connections now open) m31000| Fri Jan 18 13:15:56.291 [conn2] end connection 127.0.0.3:38111 (4 connections now open) m31000| Fri Jan 18 13:15:56.291 [initandlisten] connection accepted from 127.0.0.3:38423 #7 (5 connections now open) m31001| Fri Jan 18 13:15:56.502 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31001| Fri Jan 18 13:15:56.502 [rsBackgroundSync] SYNCED TO 127.0.0.2:31000 m31001| Fri Jan 18 13:15:56.502 [rsBackgroundSync] replSet syncing to: 127.0.0.2:31000 m31000| Fri Jan 18 13:15:56.503 [initandlisten] connection accepted from 127.0.0.3:42482 #8 (6 connections now open) m31000| Fri Jan 18 13:15:56.950 [conn3] end connection 127.0.0.4:48144 (5 connections now open) m31000| Fri Jan 18 13:15:56.950 [initandlisten] connection accepted from 127.0.0.4:59562 #9 (6 connections now open) m31001| Fri Jan 18 13:15:57.206 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 m31000| Fri Jan 18 13:15:57.206 [initandlisten] connection accepted from 127.0.0.3:49215 #10 (7 connections now open) m31002| Fri Jan 18 13:15:57.417 [rsSync] replSet initial sync pending m31002| Fri Jan 18 13:15:57.417 [rsSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31002| Fri Jan 18 13:15:57.417 [rsSync] SYNCED TO 127.0.0.2:31000 m31002| Fri Jan 18 13:15:57.417 [rsSync] replSet syncing to: 127.0.0.2:31000 m31000| Fri Jan 18 13:15:57.417 [initandlisten] connection accepted from 127.0.0.4:57365 #11 (8 connections now open) m31002| Fri Jan 18 13:15:57.418 [rsSync] build index local.me { _id: 1 } m31002| Fri Jan 18 13:15:57.419 [rsSync] build index done. scanned 0 total records. 0 secs m31002| Fri Jan 18 13:15:57.419 [rsSync] replSet initial sync drop all databases m31002| Fri Jan 18 13:15:57.419 [rsSync] dropAllDatabasesExceptLocal 1 m31002| Fri Jan 18 13:15:57.419 [rsSync] replSet initial sync clone all databases m31002| Fri Jan 18 13:15:57.420 [rsSync] replSet initial sync data copy, starting syncup m31002| Fri Jan 18 13:15:57.420 [rsSync] oplog sync 1 of 3 m31002| Fri Jan 18 13:15:57.420 [rsSync] oplog sync 2 of 3 m31002| Fri Jan 18 13:15:57.420 [rsSync] replSet initial sync building indexes m31002| Fri Jan 18 13:15:57.420 [rsSync] oplog sync 3 of 3 m31002| Fri Jan 18 13:15:57.421 [rsSync] replSet initial sync finishing up m31002| Fri Jan 18 13:15:57.421 [rsSync] replSet set minValid=50f99144:1 m31002| Fri Jan 18 13:15:57.421 [rsSync] build index local.replset.minvalid { _id: 1 } m31002| Fri Jan 18 13:15:57.422 [rsSync] build index done. scanned 0 total records. 0 secs m31002| Fri Jan 18 13:15:57.422 [rsSync] replSet initial sync done m31000| Fri Jan 18 13:15:57.423 [conn11] end connection 127.0.0.4:57365 (7 connections now open) m31000| Fri Jan 18 13:15:57.585 [conn4] end connection 127.0.0.5:40894 (6 connections now open) m31000| Fri Jan 18 13:15:57.586 [initandlisten] connection accepted from 127.0.0.5:50834 #12 (7 connections now open) m31002| Fri Jan 18 13:15:58.160 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31002| Fri Jan 18 13:15:58.160 [rsBackgroundSync] SYNCED TO 127.0.0.2:31000 m31002| Fri Jan 18 13:15:58.160 [rsBackgroundSync] replSet syncing to: 127.0.0.2:31000 m31000| Fri Jan 18 13:15:58.161 [initandlisten] connection accepted from 127.0.0.4:47928 #13 (8 connections now open) m31001| Fri Jan 18 13:15:58.206 [rsSync] replSet SECONDARY m31000| Fri Jan 18 13:15:58.208 [slaveTracking] build index local.slaves { _id: 1 } m31000| Fri Jan 18 13:15:58.209 [slaveTracking] build index done. scanned 0 total records. 0 secs m31000| Fri Jan 18 13:15:58.245 [conn5] end connection 127.0.0.6:46416 (7 connections now open) m31000| Fri Jan 18 13:15:58.245 [initandlisten] connection accepted from 127.0.0.6:33930 #14 (8 connections now open) m31004| Fri Jan 18 13:15:58.246 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state SECONDARY m31003| Fri Jan 18 13:15:58.300 [rsSync] replSet initial sync pending m31003| Fri Jan 18 13:15:58.300 [rsSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31003| Fri Jan 18 13:15:58.300 [rsSync] SYNCED TO 127.0.0.2:31000 m31003| Fri Jan 18 13:15:58.300 [rsSync] replSet syncing to: 127.0.0.2:31000 m31000| Fri Jan 18 13:15:58.301 [initandlisten] connection accepted from 127.0.0.5:56663 #15 (9 connections now open) m31003| Fri Jan 18 13:15:58.302 [rsSync] build index local.me { _id: 1 } m31003| Fri Jan 18 13:15:58.302 [rsSync] build index done. scanned 0 total records. 0 secs m31003| Fri Jan 18 13:15:58.303 [rsSync] replSet initial sync drop all databases m31003| Fri Jan 18 13:15:58.303 [rsSync] dropAllDatabasesExceptLocal 1 m31003| Fri Jan 18 13:15:58.303 [rsSync] replSet initial sync clone all databases m31003| Fri Jan 18 13:15:58.304 [rsSync] replSet initial sync data copy, starting syncup m31003| Fri Jan 18 13:15:58.304 [rsSync] oplog sync 1 of 3 m31003| Fri Jan 18 13:15:58.304 [rsSync] oplog sync 2 of 3 m31003| Fri Jan 18 13:15:58.304 [rsSync] replSet initial sync building indexes m31003| Fri Jan 18 13:15:58.304 [rsSync] oplog sync 3 of 3 m31003| Fri Jan 18 13:15:58.304 [rsSync] replSet initial sync finishing up m31003| Fri Jan 18 13:15:58.305 [rsSync] replSet set minValid=50f99144:1 m31003| Fri Jan 18 13:15:58.305 [rsSync] build index local.replset.minvalid { _id: 1 } m31003| Fri Jan 18 13:15:58.306 [rsSync] build index done. scanned 0 total records. 0 secs m31003| Fri Jan 18 13:15:58.306 [rsSync] replSet initial sync done m31000| Fri Jan 18 13:15:58.306 [conn15] end connection 127.0.0.5:56663 (8 connections now open) m31002| Fri Jan 18 13:15:58.423 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 m31000| Fri Jan 18 13:15:58.423 [initandlisten] connection accepted from 127.0.0.4:47902 #16 (9 connections now open) m31002| Fri Jan 18 13:15:58.951 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state SECONDARY m31004| Fri Jan 18 13:15:59.209 [rsSync] replSet initial sync pending m31004| Fri Jan 18 13:15:59.209 [rsSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31004| Fri Jan 18 13:15:59.209 [rsSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31004| Fri Jan 18 13:15:59.209 [rsSync] SYNCED TO 127.0.0.2:31000 m31004| Fri Jan 18 13:15:59.209 [rsSync] replSet syncing to: 127.0.0.2:31000 m31000| Fri Jan 18 13:15:59.209 [initandlisten] connection accepted from 127.0.0.6:46347 #17 (10 connections now open) m31004| Fri Jan 18 13:15:59.210 [rsSync] build index local.me { _id: 1 } m31004| Fri Jan 18 13:15:59.211 [rsSync] build index done. scanned 0 total records. 0 secs m31004| Fri Jan 18 13:15:59.211 [rsSync] replSet initial sync drop all databases m31004| Fri Jan 18 13:15:59.212 [rsSync] dropAllDatabasesExceptLocal 1 m31004| Fri Jan 18 13:15:59.212 [rsSync] replSet initial sync clone all databases m31004| Fri Jan 18 13:15:59.212 [rsSync] replSet initial sync data copy, starting syncup m31004| Fri Jan 18 13:15:59.212 [rsSync] oplog sync 1 of 3 m31004| Fri Jan 18 13:15:59.212 [rsSync] oplog sync 2 of 3 m31004| Fri Jan 18 13:15:59.212 [rsSync] replSet initial sync building indexes m31004| Fri Jan 18 13:15:59.212 [rsSync] oplog sync 3 of 3 m31004| Fri Jan 18 13:15:59.213 [rsSync] replSet initial sync finishing up m31004| Fri Jan 18 13:15:59.213 [rsSync] replSet set minValid=50f99144:1 m31004| Fri Jan 18 13:15:59.214 [rsSync] build index local.replset.minvalid { _id: 1 } m31004| Fri Jan 18 13:15:59.214 [rsSync] build index done. scanned 0 total records. 0 secs m31004| Fri Jan 18 13:15:59.214 [rsSync] replSet initial sync done m31000| Fri Jan 18 13:15:59.215 [conn17] end connection 127.0.0.6:46347 (9 connections now open) m31003| Fri Jan 18 13:15:59.237 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31003| Fri Jan 18 13:15:59.238 [rsBackgroundSync] SYNCED TO 127.0.0.2:31000 m31003| Fri Jan 18 13:15:59.238 [rsBackgroundSync] replSet syncing to: 127.0.0.2:31000 m31000| Fri Jan 18 13:15:59.238 [initandlisten] connection accepted from 127.0.0.5:57537 #18 (10 connections now open) m31003| Fri Jan 18 13:15:59.340 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 m31000| Fri Jan 18 13:15:59.341 [initandlisten] connection accepted from 127.0.0.5:47711 #19 (11 connections now open) m31002| Fri Jan 18 13:15:59.423 [rsSync] replSet SECONDARY m31003| Fri Jan 18 13:15:59.586 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state SECONDARY m31003| Fri Jan 18 13:15:59.586 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state SECONDARY m31000| Fri Jan 18 13:15:59.655 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state SECONDARY m31000| Fri Jan 18 13:15:59.655 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state SECONDARY m31004| Fri Jan 18 13:15:59.661 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31004| Fri Jan 18 13:15:59.661 [rsBackgroundSync] SYNCED TO 127.0.0.2:31000 m31004| Fri Jan 18 13:15:59.661 [rsBackgroundSync] replSet syncing to: 127.0.0.2:31000 m31000| Fri Jan 18 13:15:59.661 [initandlisten] connection accepted from 127.0.0.6:48551 #20 (12 connections now open) m31004| Fri Jan 18 13:16:00.215 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 m31000| Fri Jan 18 13:16:00.215 [initandlisten] connection accepted from 127.0.0.6:33756 #21 (13 connections now open) m31004| Fri Jan 18 13:16:00.247 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state SECONDARY m31001| Fri Jan 18 13:16:00.291 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state SECONDARY m31003| Fri Jan 18 13:16:00.306 [rsSync] replSet SECONDARY m31002| Fri Jan 18 13:16:00.951 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state SECONDARY m31004| Fri Jan 18 13:16:01.215 [rsSync] replSet SECONDARY ---- Started set! ---- { "_id" : "testReplSet", "members" : [ { "_id" : 0, "host" : "127.0.0.2:31000", "priority" : 1 }, { "_id" : 1, "host" : "127.0.0.3:31001", "priority" : 0, "slaveDelay" : 40 }, { "_id" : 2, "host" : "127.0.0.4:31002", "priority" : 0, "slaveDelay" : 40 }, { "_id" : 3, "host" : "127.0.0.5:31003", "priority" : 0, "slaveDelay" : 80 }, { "_id" : 4, "host" : "127.0.0.6:31004", "priority" : 0 } ] } Reconfiguring replica set... { "replSetReconfig" : { "_id" : "testReplSet", "members" : [ { "_id" : 0, "host" : "127.0.0.2:31000", "priority" : 1 }, { "_id" : 1, "host" : "127.0.0.3:31001", "priority" : 0, "slaveDelay" : 40 }, { "_id" : 2, "host" : "127.0.0.4:31002", "priority" : 0, "slaveDelay" : 40 }, { "_id" : 3, "host" : "127.0.0.5:31003", "priority" : 0, "slaveDelay" : 80 }, { "_id" : 4, "host" : "127.0.0.6:31004", "priority" : 0 } ], "version" : 2 } } m31000| Fri Jan 18 13:16:01.309 [conn1] replSet replSetReconfig config object parses ok, 5 members specified m31000| Fri Jan 18 13:16:01.310 [conn1] replSet replSetReconfig [2] m31000| Fri Jan 18 13:16:01.310 [conn1] replSet info saving a newer config version to local.system.replset m31000| Fri Jan 18 13:16:01.323 [conn1] replSet saveConfigLocally done m31000| Fri Jan 18 13:16:01.323 [conn1] replSet relinquishing primary state m31000| Fri Jan 18 13:16:01.323 [conn1] replSet SECONDARY m31000| Fri Jan 18 13:16:01.323 [conn1] replSet closing client sockets after relinquishing primary m31003| Fri Jan 18 13:16:01.323 [rsSyncNotifier] replset tracking exception: exception: 10278 dbclient error communicating with server: 127.0.0.2:31000 m31000| Fri Jan 18 13:16:01.323 [conn1] replSet PRIMARY m31000| Fri Jan 18 13:16:01.323 [conn19] SocketException handling request, closing client connection: 9001 socket exception [2] server [127.0.0.5:47711] m31000| Fri Jan 18 13:16:01.323 [conn1] replSet replSetReconfig new config saved locally m31002| Fri Jan 18 13:16:01.323 [rsBackgroundSync] replSet db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000 m31001| Fri Jan 18 13:16:01.323 [rsBackgroundSync] replSet db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000 Fri Jan 18 13:16:01.323 DBClientCursor::init call() failed m31001| Fri Jan 18 13:16:01.323 [rsSyncNotifier] replset tracking exception: exception: 10278 dbclient error communicating with server: 127.0.0.2:31000 m31002| Fri Jan 18 13:16:01.323 [rsSyncNotifier] replset tracking exception: exception: 10278 dbclient error communicating with server: 127.0.0.2:31000 m31000| Fri Jan 18 13:16:01.323 [conn8] SocketException handling request, closing client connection: 9001 socket exception [2] server [127.0.0.3:42482] m31003| Fri Jan 18 13:16:01.323 [rsBackgroundSync] replSet db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000 m31004| Fri Jan 18 13:16:01.323 [rsSyncNotifier] replset tracking exception: exception: 10278 dbclient error communicating with server: 127.0.0.2:31000 Caught exception error doing query: failed, this is normal after reconfig. m31004| Fri Jan 18 13:16:01.323 [rsBackgroundSync] replSet db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000 m31000| Fri Jan 18 13:16:01.323 [conn1] SocketException handling request, closing client connection: 9001 socket exception [2] server [127.0.0.1:38664] m31000| Fri Jan 18 13:16:01.323 [conn16] SocketException handling request, closing client connection: 9001 socket exception [2] server [127.0.0.4:47902] m31000| Fri Jan 18 13:16:01.323 [conn10] SocketException handling request, closing client connection: 9001 socket exception [2] server [127.0.0.3:49215] m31000| Fri Jan 18 13:16:01.323 [rsHealthPoll] replSet member 127.0.0.6:31004 is up m31000| Fri Jan 18 13:16:01.323 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state SECONDARY m31000| Fri Jan 18 13:16:01.323 [rsMgr] can't see a majority of the set, relinquishing primary m31000| Fri Jan 18 13:16:01.323 [rsMgr] replSet relinquishing primary state m31000| Fri Jan 18 13:16:01.323 [conn20] SocketException handling request, closing client connection: 9001 socket exception [2] server [127.0.0.6:48551] m31000| Fri Jan 18 13:16:01.323 [conn13] SocketException handling request, closing client connection: 9001 socket exception [2] server [127.0.0.4:47928] m31000| Fri Jan 18 13:16:01.323 [rsMgr] replSet SECONDARY m31000| Fri Jan 18 13:16:01.323 [rsHealthPoll] replSet member 127.0.0.5:31003 is up m31000| Fri Jan 18 13:16:01.323 [rsMgr] replSet closing client sockets after relinquishing primary m31000| Fri Jan 18 13:16:01.323 [rsHealthPoll] replSet member 127.0.0.4:31002 is up m31000| Fri Jan 18 13:16:01.323 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state SECONDARY m31000| Fri Jan 18 13:16:01.323 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state SECONDARY m31000| Fri Jan 18 13:16:01.324 [rsHealthPoll] replSet member 127.0.0.3:31001 is up m31000| Fri Jan 18 13:16:01.324 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state SECONDARY m31000| Fri Jan 18 13:16:01.324 [conn21] SocketException handling request, closing client connection: 9001 socket exception [2] server [127.0.0.6:33756] m31000| Fri Jan 18 13:16:01.324 [conn18] SocketException handling request, closing client connection: 9001 socket exception [2] server [127.0.0.5:57537] Fri Jan 18 13:16:01.324 trying reconnect to 127.0.0.2:31000 m31000| Fri Jan 18 13:16:01.324 [rsMgr] not electing self, 127.0.0.6:31004 would veto with '127.0.0.2:31000 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' Fri Jan 18 13:16:01.324 reconnect 127.0.0.2:31000 ok m31000| Fri Jan 18 13:16:01.324 [initandlisten] connection accepted from 127.0.0.1:38682 #22 (5 connections now open) m31000| Fri Jan 18 13:16:01.324 [rsMgr] not electing self, 127.0.0.6:31004 would veto with '127.0.0.2:31000 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Fri Jan 18 13:16:01.325 [rsMgr] not electing self, 127.0.0.6:31004 would veto with '127.0.0.2:31000 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31003| Fri Jan 18 13:16:01.587 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state SECONDARY m31003| Fri Jan 18 13:16:01.587 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31003| Fri Jan 18 13:16:01.587 [rsMgr] replset msgReceivedNewConfig version: version: 2 m31003| Fri Jan 18 13:16:01.587 [rsMgr] replSet info saving a newer config version to local.system.replset m31003| Fri Jan 18 13:16:01.591 [rsMgr] replSet saveConfigLocally done m31003| Fri Jan 18 13:16:01.591 [rsMgr] replSet replSetReconfig new config saved locally m31003| Fri Jan 18 13:16:01.591 [rsHealthPoll] replSet member 127.0.0.6:31004 is up m31003| Fri Jan 18 13:16:01.591 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state SECONDARY m31003| Fri Jan 18 13:16:01.591 [rsHealthPoll] replSet member 127.0.0.2:31000 is up m31003| Fri Jan 18 13:16:01.591 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31003| Fri Jan 18 13:16:01.591 [rsHealthPoll] replSet member 127.0.0.3:31001 is up m31003| Fri Jan 18 13:16:01.592 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state SECONDARY m31003| Fri Jan 18 13:16:01.592 [rsHealthPoll] replSet member 127.0.0.4:31002 is up m31003| Fri Jan 18 13:16:01.592 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state SECONDARY m31000| Fri Jan 18 13:16:01.655 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31000| Fri Jan 18 13:16:01.655 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31000| Fri Jan 18 13:16:01.655 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31000| Fri Jan 18 13:16:01.655 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31004| Fri Jan 18 13:16:02.246 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state SECONDARY m31004| Fri Jan 18 13:16:02.246 [rsMgr] replset msgReceivedNewConfig version: version: 2 m31004| Fri Jan 18 13:16:02.247 [rsMgr] replSet info saving a newer config version to local.system.replset m31004| Fri Jan 18 13:16:02.247 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31004| Fri Jan 18 13:16:02.281 [rsMgr] replSet saveConfigLocally done m31004| Fri Jan 18 13:16:02.281 [rsMgr] replSet replSetReconfig new config saved locally m31004| Fri Jan 18 13:16:02.281 [rsMgr] replset msgReceivedNewConfig version: version: 2 m31004| Fri Jan 18 13:16:02.281 [rsMgr] replSet info msgReceivedNewConfig but version isn't higher 2 2 m31004| Fri Jan 18 13:16:02.281 [rsHealthPoll] replSet member 127.0.0.3:31001 is up m31004| Fri Jan 18 13:16:02.281 [rsHealthPoll] replSet member 127.0.0.4:31002 is up m31004| Fri Jan 18 13:16:02.281 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state SECONDARY m31004| Fri Jan 18 13:16:02.281 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state SECONDARY m31004| Fri Jan 18 13:16:02.281 [rsHealthPoll] replSet member 127.0.0.5:31003 is up m31004| Fri Jan 18 13:16:02.282 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state SECONDARY m31004| Fri Jan 18 13:16:02.282 [rsHealthPoll] replSet member 127.0.0.2:31000 is up m31004| Fri Jan 18 13:16:02.282 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31001| Fri Jan 18 13:16:02.292 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state SECONDARY m31001| Fri Jan 18 13:16:02.292 [rsMgr] replset msgReceivedNewConfig version: version: 2 m31001| Fri Jan 18 13:16:02.292 [rsMgr] replSet info saving a newer config version to local.system.replset m31001| Fri Jan 18 13:16:02.292 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state SECONDARY m31001| Fri Jan 18 13:16:02.292 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31001| Fri Jan 18 13:16:02.305 [rsMgr] replSet saveConfigLocally done m31001| Fri Jan 18 13:16:02.305 [rsMgr] replSet replSetReconfig new config saved locally m31001| Fri Jan 18 13:16:02.305 [rsMgr] replset msgReceivedNewConfig version: version: 2 m31001| Fri Jan 18 13:16:02.305 [rsMgr] replSet info msgReceivedNewConfig but version isn't higher 2 2 m31001| Fri Jan 18 13:16:02.305 [rsMgr] replset msgReceivedNewConfig version: version: 2 m31001| Fri Jan 18 13:16:02.305 [rsHealthPoll] replSet member 127.0.0.2:31000 is up m31001| Fri Jan 18 13:16:02.305 [rsHealthPoll] replSet member 127.0.0.4:31002 is up m31001| Fri Jan 18 13:16:02.305 [rsHealthPoll] replSet member 127.0.0.6:31004 is up m31001| Fri Jan 18 13:16:02.306 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state SECONDARY m31001| Fri Jan 18 13:16:02.306 [rsHealthPoll] replSet member 127.0.0.5:31003 is up m31001| Fri Jan 18 13:16:02.305 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31001| Fri Jan 18 13:16:02.306 [rsMgr] replSet info msgReceivedNewConfig but version isn't higher 2 2 m31001| Fri Jan 18 13:16:02.306 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state SECONDARY m31001| Fri Jan 18 13:16:02.306 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state SECONDARY m31001| Fri Jan 18 13:16:02.306 [rsMgr] replSet I don't see a primary and I can't elect myself m31002| Fri Jan 18 13:16:02.951 [rsMgr] replset msgReceivedNewConfig version: version: 2 m31002| Fri Jan 18 13:16:02.951 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state SECONDARY m31002| Fri Jan 18 13:16:02.951 [rsMgr] replSet info saving a newer config version to local.system.replset m31002| Fri Jan 18 13:16:02.952 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31002| Fri Jan 18 13:16:02.972 [rsMgr] replSet saveConfigLocally done m31002| Fri Jan 18 13:16:02.973 [rsMgr] replSet replSetReconfig new config saved locally m31002| Fri Jan 18 13:16:02.973 [rsMgr] replset msgReceivedNewConfig version: version: 2 m31002| Fri Jan 18 13:16:02.973 [rsHealthPoll] replSet member 127.0.0.5:31003 is up m31002| Fri Jan 18 13:16:02.973 [rsMgr] replSet info msgReceivedNewConfig but version isn't higher 2 2 m31002| Fri Jan 18 13:16:02.973 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state SECONDARY m31002| Fri Jan 18 13:16:02.973 [rsMgr] replset msgReceivedNewConfig version: version: 2 m31002| Fri Jan 18 13:16:02.973 [rsHealthPoll] replSet member 127.0.0.3:31001 is up m31002| Fri Jan 18 13:16:02.973 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state SECONDARY m31002| Fri Jan 18 13:16:02.973 [rsMgr] replSet info msgReceivedNewConfig but version isn't higher 2 2 m31002| Fri Jan 18 13:16:02.973 [rsMgr] replset msgReceivedNewConfig version: version: 2 m31002| Fri Jan 18 13:16:02.973 [rsHealthPoll] replSet member 127.0.0.6:31004 is up m31002| Fri Jan 18 13:16:02.973 [rsHealthPoll] replSet member 127.0.0.6:31004 is now in state SECONDARY m31002| Fri Jan 18 13:16:02.973 [rsHealthPoll] replSet member 127.0.0.2:31000 is up m31002| Fri Jan 18 13:16:02.973 [rsMgr] replSet info msgReceivedNewConfig but version isn't higher 2 2 m31002| Fri Jan 18 13:16:02.973 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31004| Fri Jan 18 13:16:03.324 [conn2] end connection 127.0.0.2:55988 (4 connections now open) m31004| Fri Jan 18 13:16:03.324 [initandlisten] connection accepted from 127.0.0.2:44491 #6 (5 connections now open) m31003| Fri Jan 18 13:16:03.587 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31003| Fri Jan 18 13:16:03.587 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31003| Fri Jan 18 13:16:03.587 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31003| Fri Jan 18 13:16:03.587 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31004| Fri Jan 18 13:16:03.592 [conn5] end connection 127.0.0.5:52083 (4 connections now open) m31004| Fri Jan 18 13:16:03.592 [initandlisten] connection accepted from 127.0.0.5:59370 #7 (5 connections now open) m31004| Fri Jan 18 13:16:04.247 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31004| Fri Jan 18 13:16:04.247 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31004| Fri Jan 18 13:16:04.247 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31004| Fri Jan 18 13:16:04.247 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31003| Fri Jan 18 13:16:04.282 [conn5] end connection 127.0.0.6:42167 (4 connections now open) m31003| Fri Jan 18 13:16:04.282 [initandlisten] connection accepted from 127.0.0.6:44541 #9 (5 connections now open) m31001| Fri Jan 18 13:16:04.292 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31001| Fri Jan 18 13:16:04.292 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31001| Fri Jan 18 13:16:04.292 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31001| Fri Jan 18 13:16:04.292 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31004| Fri Jan 18 13:16:04.306 [conn3] end connection 127.0.0.3:35741 (4 connections now open) m31004| Fri Jan 18 13:16:04.306 [initandlisten] connection accepted from 127.0.0.3:60600 #8 (6 connections now open) m31002| Fri Jan 18 13:16:04.951 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31002| Fri Jan 18 13:16:04.952 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31002| Fri Jan 18 13:16:04.952 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31002| Fri Jan 18 13:16:04.952 [rsHealthPoll] Client::shutdown not called: rsHealthPoll m31004| Fri Jan 18 13:16:04.973 [conn4] end connection 127.0.0.4:37410 (4 connections now open) m31004| Fri Jan 18 13:16:04.973 [initandlisten] connection accepted from 127.0.0.4:35806 #9 (5 connections now open) m31002| Fri Jan 18 13:16:05.324 [conn3] end connection 127.0.0.2:44093 (4 connections now open) m31002| Fri Jan 18 13:16:05.324 [initandlisten] connection accepted from 127.0.0.2:33336 #10 (5 connections now open) m31001| Fri Jan 18 13:16:05.592 [conn6] end connection 127.0.0.5:52703 (4 connections now open) m31001| Fri Jan 18 13:16:05.592 [initandlisten] connection accepted from 127.0.0.5:34437 #10 (5 connections now open) m31001| Fri Jan 18 13:16:06.282 [conn8] end connection 127.0.0.6:32875 (4 connections now open) m31001| Fri Jan 18 13:16:06.282 [initandlisten] connection accepted from 127.0.0.6:49817 #11 (5 connections now open) m31002| Fri Jan 18 13:16:06.306 [conn5] end connection 127.0.0.3:41067 (4 connections now open) m31002| Fri Jan 18 13:16:06.306 [initandlisten] connection accepted from 127.0.0.3:47627 #11 (5 connections now open) m31001| Fri Jan 18 13:16:06.974 [conn4] end connection 127.0.0.4:46516 (4 connections now open) m31001| Fri Jan 18 13:16:06.974 [initandlisten] connection accepted from 127.0.0.4:54414 #12 (5 connections now open) m31000| Fri Jan 18 13:16:07.325 [rsMgr] replSet info electSelf 0 m31001| Fri Jan 18 13:16:07.325 [conn9] replSet info voting yea for 127.0.0.2:31000 (0) m31002| Fri Jan 18 13:16:07.325 [conn10] replSet info voting yea for 127.0.0.2:31000 (0) m31004| Fri Jan 18 13:16:07.325 [conn6] replSet info voting yea for 127.0.0.2:31000 (0) m31003| Fri Jan 18 13:16:07.325 [conn6] replSet info voting yea for 127.0.0.2:31000 (0) m31000| Fri Jan 18 13:16:07.654 [rsMgr] replSet PRIMARY m31004| Fri Jan 18 13:16:08.283 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state PRIMARY m31004| Fri Jan 18 13:16:08.283 [rsMgr] replSet I don't see a primary and I can't elect myself m31001| Fri Jan 18 13:16:08.307 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state PRIMARY m31002| Fri Jan 18 13:16:08.974 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state PRIMARY m31000| Fri Jan 18 13:16:09.340 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.ns, filling with zeroes... My optime : Fri Jan 18 2013 13:15:32 GMT-0500 (EST) vs Fri Jan 18 2013 13:15:32 GMT-0500 (EST) m31000| Fri Jan 18 13:16:09.500 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.ns, size: 16MB, took 0.159 secs m31000| Fri Jan 18 13:16:09.501 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.0, filling with zeroes... m31003| Fri Jan 18 13:16:09.593 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state PRIMARY m31000| Fri Jan 18 13:16:09.642 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.0, size: 16MB, took 0.141 secs m31000| Fri Jan 18 13:16:09.643 [conn22] build index foo.bar { _id: 1 } m31000| Fri Jan 18 13:16:09.644 [conn22] build index done. scanned 0 total records. 0 secs m31000| Fri Jan 18 13:16:09.644 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:304211 304ms m31002| Fri Jan 18 13:16:11.323 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31002| Fri Jan 18 13:16:11.323 [rsBackgroundSync] SYNCED TO 127.0.0.2:31000 m31002| Fri Jan 18 13:16:11.323 [rsBackgroundSync] replSet syncing to: 127.0.0.2:31000 m31001| Fri Jan 18 13:16:11.323 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31001| Fri Jan 18 13:16:11.323 [rsBackgroundSync] SYNCED TO 127.0.0.2:31000 m31001| Fri Jan 18 13:16:11.323 [rsBackgroundSync] replSet syncing to: 127.0.0.2:31000 m31003| Fri Jan 18 13:16:11.323 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31003| Fri Jan 18 13:16:11.323 [rsBackgroundSync] SYNCED TO 127.0.0.2:31000 m31003| Fri Jan 18 13:16:11.323 [rsBackgroundSync] replSet syncing to: 127.0.0.2:31000 m31004| Fri Jan 18 13:16:11.323 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.2:31000 m31004| Fri Jan 18 13:16:11.323 [rsBackgroundSync] SYNCED TO 127.0.0.2:31000 m31004| Fri Jan 18 13:16:11.323 [rsBackgroundSync] replSet syncing to: 127.0.0.2:31000 m31000| Fri Jan 18 13:16:11.324 [initandlisten] connection accepted from 127.0.0.3:38013 #23 (6 connections now open) m31000| Fri Jan 18 13:16:11.324 [initandlisten] connection accepted from 127.0.0.4:42993 #24 (7 connections now open) m31000| Fri Jan 18 13:16:11.324 [initandlisten] connection accepted from 127.0.0.5:54253 #25 (8 connections now open) m31000| Fri Jan 18 13:16:11.324 [initandlisten] connection accepted from 127.0.0.6:36483 #26 (9 connections now open) m31003| Fri Jan 18 13:16:11.324 [rsSync] replSet slavedelay sleep long time: 70 m31004| Fri Jan 18 13:16:11.325 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 m31000| Fri Jan 18 13:16:11.325 [initandlisten] connection accepted from 127.0.0.6:58138 #27 (10 connections now open) m31004| Fri Jan 18 13:16:11.326 [FileAllocator] allocating new datafile /data/db/testReplSet-4/foo.ns, filling with zeroes... m31004| Fri Jan 18 13:16:11.474 [FileAllocator] done allocating datafile /data/db/testReplSet-4/foo.ns, size: 16MB, took 0.148 secs m31004| Fri Jan 18 13:16:11.475 [FileAllocator] allocating new datafile /data/db/testReplSet-4/foo.0, filling with zeroes... m31004| Fri Jan 18 13:16:11.616 [FileAllocator] done allocating datafile /data/db/testReplSet-4/foo.0, size: 16MB, took 0.141 secs m31004| Fri Jan 18 13:16:11.618 [repl writer worker 1] build index foo.bar { _id: 1 } m31004| Fri Jan 18 13:16:11.618 [repl writer worker 1] build index done. scanned 0 total records. 0 secs My optime : Fri Jan 18 2013 13:15:32 GMT-0500 (EST) vs Fri Jan 18 2013 13:15:32 GMT-0500 (EST) m31003| Fri Jan 18 13:16:19.327 [conn6] end connection 127.0.0.2:39162 (4 connections now open) m31003| Fri Jan 18 13:16:19.327 [initandlisten] connection accepted from 127.0.0.2:41062 #10 (5 connections now open) My optime : Fri Jan 18 2013 13:15:32 GMT-0500 (EST) vs Fri Jan 18 2013 13:15:32 GMT-0500 (EST) m31002| Fri Jan 18 13:16:19.595 [conn8] end connection 127.0.0.5:52675 (4 connections now open) m31002| Fri Jan 18 13:16:19.595 [initandlisten] connection accepted from 127.0.0.5:60208 #12 (5 connections now open) m31002| Fri Jan 18 13:16:20.285 [conn9] end connection 127.0.0.6:54288 (4 connections now open) m31002| Fri Jan 18 13:16:20.285 [initandlisten] connection accepted from 127.0.0.6:55132 #13 (5 connections now open) m31003| Fri Jan 18 13:16:20.309 [conn7] end connection 127.0.0.3:36517 (4 connections now open) m31003| Fri Jan 18 13:16:20.309 [initandlisten] connection accepted from 127.0.0.3:41467 #11 (5 connections now open) m31003| Fri Jan 18 13:16:20.976 [conn8] end connection 127.0.0.4:50741 (4 connections now open) m31003| Fri Jan 18 13:16:20.976 [initandlisten] connection accepted from 127.0.0.4:53276 #12 (5 connections now open) m31001| Fri Jan 18 13:16:21.327 [conn9] end connection 127.0.0.2:58231 (4 connections now open) m31001| Fri Jan 18 13:16:21.327 [initandlisten] connection accepted from 127.0.0.2:60761 #13 (5 connections now open) m31000| Fri Jan 18 13:16:21.595 [conn12] end connection 127.0.0.5:50834 (9 connections now open) m31000| Fri Jan 18 13:16:21.595 [initandlisten] connection accepted from 127.0.0.5:43114 #28 (10 connections now open) m31000| Fri Jan 18 13:16:22.285 [conn14] end connection 127.0.0.6:33930 (9 connections now open) m31000| Fri Jan 18 13:16:22.285 [initandlisten] connection accepted from 127.0.0.6:47672 #29 (10 connections now open) m31000| Fri Jan 18 13:16:22.309 [conn7] end connection 127.0.0.3:38423 (9 connections now open) m31000| Fri Jan 18 13:16:22.309 [initandlisten] connection accepted from 127.0.0.3:49135 #30 (10 connections now open) m31000| Fri Jan 18 13:16:22.976 [conn9] end connection 127.0.0.4:59562 (9 connections now open) m31000| Fri Jan 18 13:16:22.977 [initandlisten] connection accepted from 127.0.0.4:48654 #31 (10 connections now open) My optime : Fri Jan 18 2013 13:15:32 GMT-0500 (EST) vs Fri Jan 18 2013 13:15:32 GMT-0500 (EST) My optime : Fri Jan 18 2013 13:15:32 GMT-0500 (EST) vs Fri Jan 18 2013 13:15:32 GMT-0500 (EST) m31004| Fri Jan 18 13:16:33.329 [conn6] end connection 127.0.0.2:44491 (4 connections now open) m31004| Fri Jan 18 13:16:33.329 [initandlisten] connection accepted from 127.0.0.2:41101 #10 (5 connections now open) m31004| Fri Jan 18 13:16:33.597 [conn7] end connection 127.0.0.5:59370 (4 connections now open) m31004| Fri Jan 18 13:16:33.598 [initandlisten] connection accepted from 127.0.0.5:34288 #11 (5 connections now open) m31003| Fri Jan 18 13:16:34.287 [conn9] end connection 127.0.0.6:44541 (4 connections now open) m31003| Fri Jan 18 13:16:34.288 [initandlisten] connection accepted from 127.0.0.6:39815 #13 (6 connections now open) m31004| Fri Jan 18 13:16:34.311 [conn8] end connection 127.0.0.3:60600 (4 connections now open) m31004| Fri Jan 18 13:16:34.312 [initandlisten] connection accepted from 127.0.0.3:58257 #12 (5 connections now open) My optime : Fri Jan 18 2013 13:15:32 GMT-0500 (EST) vs Fri Jan 18 2013 13:15:32 GMT-0500 (EST) m31004| Fri Jan 18 13:16:34.979 [conn9] end connection 127.0.0.4:35806 (4 connections now open) m31004| Fri Jan 18 13:16:34.979 [initandlisten] connection accepted from 127.0.0.4:46981 #13 (5 connections now open) m31002| Fri Jan 18 13:16:35.329 [conn10] end connection 127.0.0.2:33336 (4 connections now open) m31002| Fri Jan 18 13:16:35.329 [initandlisten] connection accepted from 127.0.0.2:59995 #14 (5 connections now open) m31001| Fri Jan 18 13:16:35.598 [conn10] end connection 127.0.0.5:34437 (4 connections now open) m31001| Fri Jan 18 13:16:35.598 [initandlisten] connection accepted from 127.0.0.5:41209 #14 (5 connections now open) m31001| Fri Jan 18 13:16:36.288 [conn11] end connection 127.0.0.6:49817 (4 connections now open) m31001| Fri Jan 18 13:16:36.288 [initandlisten] connection accepted from 127.0.0.6:60047 #15 (5 connections now open) m31002| Fri Jan 18 13:16:36.312 [conn11] end connection 127.0.0.3:47627 (4 connections now open) m31002| Fri Jan 18 13:16:36.312 [initandlisten] connection accepted from 127.0.0.3:36977 #15 (5 connections now open) m31001| Fri Jan 18 13:16:36.979 [conn12] end connection 127.0.0.4:54414 (4 connections now open) m31001| Fri Jan 18 13:16:36.979 [initandlisten] connection accepted from 127.0.0.4:58034 #16 (5 connections now open) My optime : Fri Jan 18 2013 13:15:32 GMT-0500 (EST) vs Fri Jan 18 2013 13:15:32 GMT-0500 (EST) m31001| Fri Jan 18 13:16:41.325 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 m31002| Fri Jan 18 13:16:41.325 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 m31000| Fri Jan 18 13:16:41.325 [initandlisten] connection accepted from 127.0.0.3:45231 #32 (11 connections now open) m31000| Fri Jan 18 13:16:41.325 [initandlisten] connection accepted from 127.0.0.4:57465 #33 (12 connections now open) My optime : Fri Jan 18 2013 13:15:32 GMT-0500 (EST) vs Fri Jan 18 2013 13:16:01 GMT-0500 (EST) Running: sudo bash -c tc class add dev lo parent 1:1 classid 1:11 htb rate 1bps ceil 1000Mbps > /tmp/output Fri Jan 18 13:16:44.359 shell: started program sudo bash -c tc class add dev lo parent 1:1 classid 1:11 htb rate 1bps ceil 1000Mbps > /tmp/output Running: sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.2 match ip dst 127.0.0.5 flowid 1:11 > /tmp/output Fri Jan 18 13:16:44.374 shell: started program sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.2 match ip dst 127.0.0.5 flowid 1:11 > /tmp/output Running: sudo bash -c tc qdisc add dev lo parent 1:11 handle 11: netem > /tmp/output Fri Jan 18 13:16:44.388 shell: started program sudo bash -c tc qdisc add dev lo parent 1:11 handle 11: netem > /tmp/output Running: sudo bash -c tc qdisc change dev lo parent 1:11 handle 11: netem delay 40000ms > /tmp/output Fri Jan 18 13:16:44.403 shell: started program sudo bash -c tc qdisc change dev lo parent 1:11 handle 11: netem delay 40000ms > /tmp/output Running: sudo bash -c tc class add dev lo parent 1:1 classid 1:12 htb rate 1bps ceil 1000Mbps > /tmp/output Fri Jan 18 13:16:44.416 shell: started program sudo bash -c tc class add dev lo parent 1:1 classid 1:12 htb rate 1bps ceil 1000Mbps > /tmp/output Running: sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.5 match ip dst 127.0.0.2 flowid 1:12 > /tmp/output Fri Jan 18 13:16:44.429 shell: started program sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.5 match ip dst 127.0.0.2 flowid 1:12 > /tmp/output Running: sudo bash -c tc qdisc add dev lo parent 1:12 handle 12: netem > /tmp/output Fri Jan 18 13:16:44.443 shell: started program sudo bash -c tc qdisc add dev lo parent 1:12 handle 12: netem > /tmp/output Running: sudo bash -c tc qdisc change dev lo parent 1:12 handle 12: netem delay 40000ms > /tmp/output Fri Jan 18 13:16:44.457 shell: started program sudo bash -c tc qdisc change dev lo parent 1:12 handle 12: netem delay 40000ms > /tmp/output Running: sudo bash -c dig +short undefined > /tmp/output Fri Jan 18 13:16:44.471 shell: started program sudo bash -c dig +short undefined > /tmp/output Running: sudo bash -c tc class add dev lo parent 1:1 classid 1:13 htb rate 1bps ceil 1000Mbps > /tmp/output Fri Jan 18 13:16:44.500 shell: started program sudo bash -c tc class add dev lo parent 1:1 classid 1:13 htb rate 1bps ceil 1000Mbps > /tmp/output Running: sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.5 match ip dport 31001 0xFFFF flowid 1:13 > /tmp/output Fri Jan 18 13:16:44.512 shell: started program sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.5 match ip dport 31001 0xFFFF flowid 1:13 > /tmp/output Running: sudo bash -c tc qdisc add dev lo parent 1:13 handle 13: netem > /tmp/output Fri Jan 18 13:16:44.526 shell: started program sudo bash -c tc qdisc add dev lo parent 1:13 handle 13: netem > /tmp/output Running: sudo bash -c tc qdisc change dev lo parent 1:13 handle 13: netem delay 0ms 100ms > /tmp/output Fri Jan 18 13:16:44.534 shell: started program sudo bash -c tc qdisc change dev lo parent 1:13 handle 13: netem delay 0ms 100ms > /tmp/output Running: sudo bash -c dig +short undefined > /tmp/output Fri Jan 18 13:16:44.541 shell: started program sudo bash -c dig +short undefined > /tmp/output Running: sudo bash -c tc class add dev lo parent 1:1 classid 1:14 htb rate 1bps ceil 1000Mbps > /tmp/output Fri Jan 18 13:16:44.562 shell: started program sudo bash -c tc class add dev lo parent 1:1 classid 1:14 htb rate 1bps ceil 1000Mbps > /tmp/output Running: sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.5 match ip dport 31002 0xFFFF flowid 1:14 > /tmp/output Fri Jan 18 13:16:44.569 shell: started program sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.5 match ip dport 31002 0xFFFF flowid 1:14 > /tmp/output Running: sudo bash -c tc qdisc add dev lo parent 1:14 handle 14: netem > /tmp/output Fri Jan 18 13:16:44.577 shell: started program sudo bash -c tc qdisc add dev lo parent 1:14 handle 14: netem > /tmp/output Running: sudo bash -c tc qdisc change dev lo parent 1:14 handle 14: netem delay 0ms 100ms > /tmp/output Fri Jan 18 13:16:44.585 shell: started program sudo bash -c tc qdisc change dev lo parent 1:14 handle 14: netem delay 0ms 100ms > /tmp/output Running: sudo bash -c dig +short undefined > /tmp/output Fri Jan 18 13:16:44.596 shell: started program sudo bash -c dig +short undefined > /tmp/output Running: sudo bash -c tc class add dev lo parent 1:1 classid 1:15 htb rate 1bps ceil 1000Mbps > /tmp/output Fri Jan 18 13:16:44.625 shell: started program sudo bash -c tc class add dev lo parent 1:1 classid 1:15 htb rate 1bps ceil 1000Mbps > /tmp/output Running: sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.5 match ip dport 31004 0xFFFF flowid 1:15 > /tmp/output Fri Jan 18 13:16:44.640 shell: started program sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.5 match ip dport 31004 0xFFFF flowid 1:15 > /tmp/output Running: sudo bash -c tc qdisc add dev lo parent 1:15 handle 15: netem > /tmp/output Fri Jan 18 13:16:44.653 shell: started program sudo bash -c tc qdisc add dev lo parent 1:15 handle 15: netem > /tmp/output Running: sudo bash -c tc qdisc change dev lo parent 1:15 handle 15: netem delay 500ms > /tmp/output Fri Jan 18 13:16:44.668 shell: started program sudo bash -c tc qdisc change dev lo parent 1:15 handle 15: netem delay 500ms > /tmp/output Running: sudo bash -c tc -s -d -iec qdisc show dev lo > /tmp/output Fri Jan 18 13:16:44.682 shell: started program sudo bash -c tc -s -d -iec qdisc show dev lo > /tmp/output qdisc htb 1: root refcnt 2 r2q 10 default 0 direct_packets_stat 4821 ver 3.17 Sent 829377 bytes 4821 pkt (dropped 2, overlimits 0 requeues 0) backlog 0b 0p requeues 0 qdisc netem 11: parent 1:11 limit 1000 delay 40.0s Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) backlog 0b 0p requeues 0 qdisc netem 12: parent 1:12 limit 1000 delay 40.0s Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) backlog 0b 0p requeues 0 qdisc netem 13: parent 1:13 limit 1000 Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) backlog 0b 0p requeues 0 qdisc netem 14: parent 1:14 limit 1000 Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) backlog 0b 0p requeues 0 qdisc netem 15: parent 1:15 limit 1000 delay 500.0ms Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) backlog 0b 0p requeues 0 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 79, "optime" : { "t" : 1358533006000, "i" : 8904 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 45, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:46Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 45, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 45, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:43Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 45, "optime" : { "t" : 1358533005000, "i" : 2360 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:46Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 44, "optime" : { "t" : 1358533006000, "i" : 1671 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 79, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 44, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 44, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 44, "optime" : { "t" : 1358533006000, "i" : 1671 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 44, "optime" : { "t" : 1358533004000, "i" : 2476 }, "optimeDate" : ISODate("2013-01-18T18:16:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 44, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:46Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 78, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 44, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:45Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 44, "optime" : { "t" : 1358533004000, "i" : 2447 }, "optimeDate" : ISODate("2013-01-18T18:16:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:46Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 45, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 45, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:46Z"), "pingMs" : 91, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 45, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 45, "optime" : { "t" : 1358533006000, "i" : 12 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:46Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 44, "optime" : { "t" : 1358533006000, "i" : 1671 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 44, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:46Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 44, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 44, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358533006000, "i" : 8647 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.5:31003", "127.0.0.6:31004" ] } } m31001| Fri Jan 18 13:16:49.325 [FileAllocator] allocating new datafile /data/db/testReplSet-1/foo.ns, filling with zeroes... m31002| Fri Jan 18 13:16:49.326 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.ns, filling with zeroes... m31002| Fri Jan 18 13:16:49.497 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.ns, size: 16MB, took 0.17 secs m31002| Fri Jan 18 13:16:49.497 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.0, filling with zeroes... m31001| Fri Jan 18 13:16:49.513 [FileAllocator] done allocating datafile /data/db/testReplSet-1/foo.ns, size: 16MB, took 0.187 secs m31001| Fri Jan 18 13:16:49.513 [FileAllocator] allocating new datafile /data/db/testReplSet-1/foo.0, filling with zeroes... m31002| Fri Jan 18 13:16:49.616 [conn12] end connection 127.0.0.5:60208 (4 connections now open) m31002| Fri Jan 18 13:16:49.647 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.0, size: 16MB, took 0.149 secs m31002| Fri Jan 18 13:16:49.648 [repl writer worker 1] build index foo.bar { _id: 1 } m31002| Fri Jan 18 13:16:49.649 [repl writer worker 1] build index done. scanned 0 total records. 0 secs m31001| Fri Jan 18 13:16:49.689 [FileAllocator] done allocating datafile /data/db/testReplSet-1/foo.0, size: 16MB, took 0.175 secs m31001| Fri Jan 18 13:16:49.690 [repl writer worker 1] build index foo.bar { _id: 1 } m31001| Fri Jan 18 13:16:49.691 [repl writer worker 1] build index done. scanned 0 total records. 0 secs m31002| Fri Jan 18 13:16:49.750 [initandlisten] connection accepted from 127.0.0.5:35863 #16 (5 connections now open) m31002| Fri Jan 18 13:16:50.290 [conn13] end connection 127.0.0.6:55132 (4 connections now open) m31002| Fri Jan 18 13:16:50.290 [initandlisten] connection accepted from 127.0.0.6:34892 #17 (5 connections now open) m31003| Fri Jan 18 13:16:50.314 [conn11] end connection 127.0.0.3:41467 (4 connections now open) m31003| Fri Jan 18 13:16:50.314 [initandlisten] connection accepted from 127.0.0.3:43123 #14 (5 connections now open) m31003| Fri Jan 18 13:16:50.981 [conn12] end connection 127.0.0.4:53276 (4 connections now open) m31003| Fri Jan 18 13:16:50.981 [initandlisten] connection accepted from 127.0.0.4:34370 #15 (5 connections now open) m31001| Fri Jan 18 13:16:51.332 [conn13] end connection 127.0.0.2:60761 (4 connections now open) m31001| Fri Jan 18 13:16:51.332 [initandlisten] connection accepted from 127.0.0.2:48727 #17 (5 connections now open) m31000| Fri Jan 18 13:16:52.290 [conn29] end connection 127.0.0.6:47672 (11 connections now open) m31000| Fri Jan 18 13:16:52.290 [initandlisten] connection accepted from 127.0.0.6:33706 #34 (12 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:16:52.314 [conn30] end connection 127.0.0.3:49135 (11 connections now open) m31000| Fri Jan 18 13:16:52.314 [initandlisten] connection accepted from 127.0.0.3:60858 #35 (12 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 85, "optime" : { "t" : 1358533012000, "i" : 2904 }, "optimeDate" : ISODate("2013-01-18T18:16:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 51, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 51, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 51, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:43Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 51, "optime" : { "t" : 1358533011000, "i" : 2633 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 50, "optime" : { "t" : 1358533012000, "i" : 2630 }, "optimeDate" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 85, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358533012000, "i" : 2526 }, "optimeDate" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 50, "optime" : { "t" : 1358533010000, "i" : 8243 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:51Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 84, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:51Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358533010000, "i" : 8100 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 51, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 51, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:52Z"), "pingMs" : 65, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 51, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 27, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 83, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 51, "optime" : { "t" : 1358533011000, "i" : 1048 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:52Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 50, "optime" : { "t" : 1358533012000, "i" : 2630 }, "optimeDate" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 83, "optime" : { "t" : 1358533012000, "i" : 2904 }, "optimeDate" : ISODate("2013-01-18T18:16:52Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.5:31003", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:16:52.981 [conn31] end connection 127.0.0.4:48654 (11 connections now open) m31000| Fri Jan 18 13:16:52.981 [initandlisten] connection accepted from 127.0.0.4:37334 #36 (12 connections now open) m31000| Fri Jan 18 13:16:55.331 [rsHealthPoll] DBClientCursor::init call() failed m31000| Fri Jan 18 13:16:55.331 [rsHealthPoll] replSet info 127.0.0.5:31003 is down (or slow to respond): m31000| Fri Jan 18 13:16:55.331 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state DOWN m31003| Fri Jan 18 13:16:55.599 [rsHealthPoll] DBClientCursor::init call() failed m31003| Fri Jan 18 13:16:55.599 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31003| Fri Jan 18 13:16:56.599 [rsHealthPoll] replSet info 127.0.0.2:31000 is down (or slow to respond): m31003| Fri Jan 18 13:16:56.599 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state DOWN STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 90, "optime" : { "t" : 1358533017000, "i" : 5098 }, "optimeDate" : ISODate("2013-01-18T18:16:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 56, "optime" : { "t" : 1358532974000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 56, "optime" : { "t" : 1358532974000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 56, "optime" : { "t" : 1358533017000, "i" : 2854 }, "optimeDate" : ISODate("2013-01-18T18:16:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 55, "optime" : { "t" : 1358533016000, "i" : 3344 }, "optimeDate" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 90, "optime" : { "t" : 1358532974000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:14Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358532974000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:56Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358533016000, "i" : 3334 }, "optimeDate" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 55, "optime" : { "t" : 1358533016000, "i" : 9233 }, "optimeDate" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358532974000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 89, "optime" : { "t" : 1358532974000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:14Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358533016000, "i" : 8397 }, "optimeDate" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 56, "optime" : { "t" : 1358532974000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:56Z"), "pingMs" : 56, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 56, "optime" : { "t" : 1358532974000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:56Z"), "pingMs" : 28, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 88, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 56, "optime" : { "t" : 1358533015000, "i" : 5472 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:56Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:16:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 55, "optime" : { "t" : 1358533016000, "i" : 3180 }, "optimeDate" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358532974000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:56Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358532974000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:16:56Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 88, "optime" : { "t" : 1358533017000, "i" : 5098 }, "optimeDate" : ISODate("2013-01-18T18:16:57Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.5:31003", "127.0.0.6:31004" ] } } STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 95, "optime" : { "t" : 1358533022000, "i" : 8424 }, "optimeDate" : ISODate("2013-01-18T18:17:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 61, "optime" : { "t" : 1358532979000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:02Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 61, "optime" : { "t" : 1358532979000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 61, "optime" : { "t" : 1358533021000, "i" : 2730 }, "optimeDate" : ISODate("2013-01-18T18:17:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:02Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 60, "optime" : { "t" : 1358533022000, "i" : 2496 }, "optimeDate" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 95, "optime" : { "t" : 1358532979000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:19Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358532979000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358533022000, "i" : 2496 }, "optimeDate" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 60, "optime" : { "t" : 1358533020000, "i" : 2914 }, "optimeDate" : ISODate("2013-01-18T18:17:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358532979000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:02Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 94, "optime" : { "t" : 1358532979000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:19Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:01Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358533020000, "i" : 2914 }, "optimeDate" : ISODate("2013-01-18T18:17:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:02Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 61, "optime" : { "t" : 1358532979000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:02Z"), "pingMs" : 44, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 61, "optime" : { "t" : 1358532979000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 29, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 93, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 61, "optime" : { "t" : 1358533021000, "i" : 1149 }, "optimeDate" : ISODate("2013-01-18T18:17:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:02Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 60, "optime" : { "t" : 1358533022000, "i" : 2496 }, "optimeDate" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358532979000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:02Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358532979000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 93, "optime" : { "t" : 1358533022000, "i" : 8262 }, "optimeDate" : ISODate("2013-01-18T18:17:02Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.5:31003", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:17:03.334 [conn10] end connection 127.0.0.2:41101 (4 connections now open) m31004| Fri Jan 18 13:17:03.334 [initandlisten] connection accepted from 127.0.0.2:60265 #14 (5 connections now open) m31003| Fri Jan 18 13:17:04.292 [conn13] end connection 127.0.0.6:39815 (4 connections now open) m31003| Fri Jan 18 13:17:04.292 [initandlisten] connection accepted from 127.0.0.6:47246 #16 (5 connections now open) m31004| Fri Jan 18 13:17:04.316 [conn12] end connection 127.0.0.3:58257 (4 connections now open) m31004| Fri Jan 18 13:17:04.316 [initandlisten] connection accepted from 127.0.0.3:36951 #15 (5 connections now open) m31004| Fri Jan 18 13:17:04.983 [conn13] end connection 127.0.0.4:46981 (4 connections now open) m31004| Fri Jan 18 13:17:04.983 [initandlisten] connection accepted from 127.0.0.4:41408 #16 (5 connections now open) m31002| Fri Jan 18 13:17:05.334 [conn14] end connection 127.0.0.2:59995 (4 connections now open) m31002| Fri Jan 18 13:17:05.334 [initandlisten] connection accepted from 127.0.0.2:40402 #18 (5 connections now open) m31001| Fri Jan 18 13:17:06.073 [conn14] end connection 127.0.0.5:41209 (4 connections now open) m31001| Fri Jan 18 13:17:06.073 [initandlisten] connection accepted from 127.0.0.5:38588 #18 (5 connections now open) m31003| Fri Jan 18 13:17:06.102 [rsMgr] replSet I don't see a primary and I can't elect myself m31001| Fri Jan 18 13:17:06.292 [conn15] end connection 127.0.0.6:60047 (4 connections now open) m31001| Fri Jan 18 13:17:06.292 [initandlisten] connection accepted from 127.0.0.6:36221 #19 (5 connections now open) m31002| Fri Jan 18 13:17:06.316 [conn15] end connection 127.0.0.3:36977 (4 connections now open) m31002| Fri Jan 18 13:17:06.316 [initandlisten] connection accepted from 127.0.0.3:50464 #19 (5 connections now open) m31001| Fri Jan 18 13:17:06.983 [conn16] end connection 127.0.0.4:58034 (4 connections now open) m31001| Fri Jan 18 13:17:06.984 [initandlisten] connection accepted from 127.0.0.4:48702 #20 (5 connections now open) m31000| Fri Jan 18 13:17:07.331 [rsHealthPoll] replset info 127.0.0.5:31003 heartbeat failed, retrying m31000| Fri Jan 18 13:17:07.773 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.1, filling with zeroes... m31000| Fri Jan 18 13:17:08.046 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.1, size: 32MB, took 0.272 secs m31000| Fri Jan 18 13:17:08.046 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:273215 273ms m31004| Fri Jan 18 13:17:08.047 [FileAllocator] allocating new datafile /data/db/testReplSet-4/foo.1, filling with zeroes... STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 101, "optime" : { "t" : 1358533028000, "i" : 4852 }, "optimeDate" : ISODate("2013-01-18T18:17:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 67, "optime" : { "t" : 1358532984000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 67, "optime" : { "t" : 1358532984000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 67, "optime" : { "t" : 1358533027000, "i" : 2972 }, "optimeDate" : ISODate("2013-01-18T18:17:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 66, "optime" : { "t" : 1358533026000, "i" : 2580 }, "optimeDate" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:07Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 101, "optime" : { "t" : 1358532984000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:24Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 66, "optime" : { "t" : 1358532984000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:06Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 66, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:08Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 66, "optime" : { "t" : 1358533026000, "i" : 2537 }, "optimeDate" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 66, "optime" : { "t" : 1358533026000, "i" : 8162 }, "optimeDate" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:07Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 66, "optime" : { "t" : 1358532984000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 100, "optime" : { "t" : 1358532984000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:24Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 66, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:08Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 66, "optime" : { "t" : 1358533026000, "i" : 8162 }, "optimeDate" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 67, "optime" : { "t" : 1358532984000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 45, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 67, "optime" : { "t" : 1358532984000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 26, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 99, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 67, "optime" : { "t" : 1358533026000, "i" : 828 }, "optimeDate" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:06Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 66, "optime" : { "t" : 1358533026000, "i" : 2399 }, "optimeDate" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:07Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 66, "optime" : { "t" : 1358532984000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:06Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 66, "optime" : { "t" : 1358532984000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:06Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 66, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 99, "optime" : { "t" : 1358533027000, "i" : 6877 }, "optimeDate" : ISODate("2013-01-18T18:17:07Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.5:31003", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:17:08.338 [FileAllocator] done allocating datafile /data/db/testReplSet-4/foo.1, size: 32MB, took 0.291 secs m31003| Fri Jan 18 13:17:08.599 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31004| Fri Jan 18 13:17:08.602 [conn11] end connection 127.0.0.5:34288 (4 connections now open) m31004| Fri Jan 18 13:17:09.103 [initandlisten] connection accepted from 127.0.0.5:50113 #17 (5 connections now open) m31000| Fri Jan 18 13:17:11.487 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:343474 171ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 106, "optime" : { "t" : 1358533033000, "i" : 4690 }, "optimeDate" : ISODate("2013-01-18T18:17:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 72, "optime" : { "t" : 1358532989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 72, "optime" : { "t" : 1358532989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 72, "optime" : { "t" : 1358533033000, "i" : 2595 }, "optimeDate" : ISODate("2013-01-18T18:17:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 71, "optime" : { "t" : 1358533032000, "i" : 2641 }, "optimeDate" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 106, "optime" : { "t" : 1358532989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:29Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358532989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:12Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358533032000, "i" : 2613 }, "optimeDate" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 71, "optime" : { "t" : 1358533032000, "i" : 8397 }, "optimeDate" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358532989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 105, "optime" : { "t" : 1358532989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:29Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358533032000, "i" : 8389 }, "optimeDate" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 72, "optime" : { "t" : 1358532989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:12Z"), "pingMs" : 38, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 72, "optime" : { "t" : 1358532989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:12Z"), "pingMs" : 16, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 104, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 72, "optime" : { "t" : 1358533031000, "i" : 2926 }, "optimeDate" : ISODate("2013-01-18T18:17:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:12Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 71, "optime" : { "t" : 1358533032000, "i" : 2125 }, "optimeDate" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358532989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:12Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358532989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:12Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 104, "optime" : { "t" : 1358533033000, "i" : 4690 }, "optimeDate" : ISODate("2013-01-18T18:17:13Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.5:31003", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:17:14.355 [rsBackgroundSync] Socket recv() timeout 127.0.0.2:31000 m31003| Fri Jan 18 13:17:14.355 [rsBackgroundSync] SocketException: remote: 127.0.0.2:31000 error: 9001 socket exception [3] server [127.0.0.2:31000] m31003| Fri Jan 18 13:17:14.355 [rsBackgroundSync] replSet db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 112, "optime" : { "t" : 1358533039000, "i" : 95 }, "optimeDate" : ISODate("2013-01-18T18:17:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 78, "optime" : { "t" : 1358532994000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:18Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 78, "optime" : { "t" : 1358532994000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:18Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 78, "optime" : { "t" : 1358533037000, "i" : 2533 }, "optimeDate" : ISODate("2013-01-18T18:17:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:18Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 77, "optime" : { "t" : 1358533038000, "i" : 2870 }, "optimeDate" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 112, "optime" : { "t" : 1358532994000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:34Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358532994000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:18Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358533038000, "i" : 2582 }, "optimeDate" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 77, "optime" : { "t" : 1358533038000, "i" : 8312 }, "optimeDate" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358532994000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 111, "optime" : { "t" : 1358532994000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:34Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358533038000, "i" : 8311 }, "optimeDate" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 78, "optime" : { "t" : 1358532994000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:18Z"), "pingMs" : 20, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 78, "optime" : { "t" : 1358532994000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:18Z"), "pingMs" : 19, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 110, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "errmsg" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 78, "optime" : { "t" : 1358533033000, "i" : 4690 }, "optimeDate" : ISODate("2013-01-18T18:17:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:18Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 77, "optime" : { "t" : 1358533038000, "i" : 2386 }, "optimeDate" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358532994000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:18Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358532994000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:18Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 110, "optime" : { "t" : 1358533039000, "i" : 95 }, "optimeDate" : ISODate("2013-01-18T18:17:19Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.5:31003", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:17:19.772 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:118686 nreturned:140001 reslen:2380037 118ms m31000| Fri Jan 18 13:17:19.807 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:112271 nreturned:140001 reslen:2380037 112ms m31002| Fri Jan 18 13:17:20.106 [conn16] end connection 127.0.0.5:35863 (4 connections now open) m31002| Fri Jan 18 13:17:20.130 [initandlisten] connection accepted from 127.0.0.5:42789 #20 (5 connections now open) m31002| Fri Jan 18 13:17:20.294 [conn17] end connection 127.0.0.6:34892 (4 connections now open) m31002| Fri Jan 18 13:17:20.295 [initandlisten] connection accepted from 127.0.0.6:36751 #21 (5 connections now open) m31003| Fri Jan 18 13:17:20.318 [conn14] end connection 127.0.0.3:43123 (4 connections now open) m31003| Fri Jan 18 13:17:20.318 [initandlisten] connection accepted from 127.0.0.3:48138 #17 (5 connections now open) m31000| Fri Jan 18 13:17:20.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:17:20.986 [conn15] end connection 127.0.0.4:34370 (4 connections now open) m31003| Fri Jan 18 13:17:20.986 [initandlisten] connection accepted from 127.0.0.4:35385 #18 (5 connections now open) m31001| Fri Jan 18 13:17:21.337 [conn17] end connection 127.0.0.2:48727 (4 connections now open) m31001| Fri Jan 18 13:17:21.337 [initandlisten] connection accepted from 127.0.0.2:44458 #21 (5 connections now open) m31003| Fri Jan 18 13:17:21.599 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31000| Fri Jan 18 13:17:22.295 [conn34] end connection 127.0.0.6:33706 (11 connections now open) m31000| Fri Jan 18 13:17:22.295 [initandlisten] connection accepted from 127.0.0.6:46140 #37 (12 connections now open) m31000| Fri Jan 18 13:17:22.318 [conn35] end connection 127.0.0.3:60858 (11 connections now open) m31000| Fri Jan 18 13:17:22.318 [initandlisten] connection accepted from 127.0.0.3:41334 #38 (12 connections now open) m31000| Fri Jan 18 13:17:22.986 [conn36] end connection 127.0.0.4:37334 (11 connections now open) m31000| Fri Jan 18 13:17:22.986 [initandlisten] connection accepted from 127.0.0.4:39762 #39 (12 connections now open) m31003| Fri Jan 18 13:17:23.326 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 117, "optime" : { "t" : 1358533044000, "i" : 2063 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 83, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 83, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 83, "optime" : { "t" : 1358533043000, "i" : 3915 }, "optimeDate" : ISODate("2013-01-18T18:17:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 82, "optime" : { "t" : 1358533042000, "i" : 2874 }, "optimeDate" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:23Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 117, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 82, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:22Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 82, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 82, "optime" : { "t" : 1358533042000, "i" : 2735 }, "optimeDate" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 82, "optime" : { "t" : 1358533042000, "i" : 7473 }, "optimeDate" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:23Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 82, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 116, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 82, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:24Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 82, "optime" : { "t" : 1358533042000, "i" : 7285 }, "optimeDate" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 m31000| Fri Jan 18 13:17:30.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31004| Fri Jan 18 13:17:33.339 [conn14] end connection 127.0.0.2:60265 (4 connections now open) m31004| Fri Jan 18 13:17:33.339 [initandlisten] connection accepted from 127.0.0.2:53977 #18 (6 connections now open) m31003| Fri Jan 18 13:17:34.297 [conn16] end connection 127.0.0.6:47246 (4 connections now open) m31003| Fri Jan 18 13:17:34.297 [initandlisten] connection accepted from 127.0.0.6:48105 #19 (5 connections now open) m31004| Fri Jan 18 13:17:34.320 [conn15] end connection 127.0.0.3:36951 (4 connections now open) m31004| Fri Jan 18 13:17:34.320 [initandlisten] connection accepted from 127.0.0.3:40104 #19 (5 connections now open) m31003| Fri Jan 18 13:17:34.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31004| Fri Jan 18 13:17:34.988 [conn16] end connection 127.0.0.4:41408 (4 connections now open) m31004| Fri Jan 18 13:17:34.988 [initandlisten] connection accepted from 127.0.0.4:47240 #20 (5 connections now open) m31002| Fri Jan 18 13:17:35.339 [conn18] end connection 127.0.0.2:40402 (4 connections now open) m31002| Fri Jan 18 13:17:35.339 [initandlisten] connection accepted from 127.0.0.2:37200 #22 (5 connections now open) m31000| Fri Jan 18 13:17:35.599 [conn28] end connection 127.0.0.5:43114 (11 connections now open) m31001| Fri Jan 18 13:17:36.297 [conn19] end connection 127.0.0.6:36221 (4 connections now open) m31001| Fri Jan 18 13:17:36.297 [initandlisten] connection accepted from 127.0.0.6:35381 #22 (5 connections now open) m31002| Fri Jan 18 13:17:36.321 [conn19] end connection 127.0.0.3:50464 (4 connections now open) m31002| Fri Jan 18 13:17:36.321 [initandlisten] connection accepted from 127.0.0.3:49738 #23 (5 connections now open) m31001| Fri Jan 18 13:17:36.378 [conn18] end connection 127.0.0.5:38588 (4 connections now open) m31001| Fri Jan 18 13:17:36.429 [initandlisten] connection accepted from 127.0.0.5:48013 #23 (5 connections now open) m31001| Fri Jan 18 13:17:36.988 [conn20] end connection 127.0.0.4:48702 (4 connections now open) m31001| Fri Jan 18 13:17:36.989 [initandlisten] connection accepted from 127.0.0.4:47513 #24 (5 connections now open) m31003| Fri Jan 18 13:17:37.331 [conn10] end connection 127.0.0.2:41062 (4 connections now open) m31003| Fri Jan 18 13:17:44.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31004| Fri Jan 18 13:17:46.608 [conn17] end connection 127.0.0.5:50113 (4 connections now open) m31004| Fri Jan 18 13:17:47.108 [initandlisten] connection accepted from 127.0.0.5:33714 #21 (5 connections now open) m31001| Fri Jan 18 13:17:48.396 [FileAllocator] allocating new datafile /data/db/testReplSet-1/foo.1, filling with zeroes... m31002| Fri Jan 18 13:17:48.595 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.1, filling with zeroes... m31001| Fri Jan 18 13:17:48.687 [FileAllocator] done allocating datafile /data/db/testReplSet-1/foo.1, size: 32MB, took 0.29 secs m31002| Fri Jan 18 13:17:48.870 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.1, size: 32MB, took 0.275 secs m31002| Fri Jan 18 13:17:50.299 [conn21] end connection 127.0.0.6:36751 (4 connections now open) m31002| Fri Jan 18 13:17:50.299 [initandlisten] connection accepted from 127.0.0.6:54848 #24 (6 connections now open) m31003| Fri Jan 18 13:17:50.323 [conn17] end connection 127.0.0.3:48138 (3 connections now open) m31003| Fri Jan 18 13:17:50.323 [initandlisten] connection accepted from 127.0.0.3:55797 #20 (4 connections now open) m31002| Fri Jan 18 13:17:50.592 [conn20] end connection 127.0.0.5:42789 (4 connections now open) m31002| Fri Jan 18 13:17:50.647 [initandlisten] connection accepted from 127.0.0.5:35748 #25 (5 connections now open) m31003| Fri Jan 18 13:17:50.991 [conn18] end connection 127.0.0.4:35385 (3 connections now open) m31003| Fri Jan 18 13:17:50.991 [initandlisten] connection accepted from 127.0.0.4:43606 #21 (4 connections now open) m31001| Fri Jan 18 13:17:51.342 [conn21] end connection 127.0.0.2:44458 (4 connections now open) m31001| Fri Jan 18 13:17:51.342 [initandlisten] connection accepted from 127.0.0.2:58484 #25 (5 connections now open) m31000| Fri Jan 18 13:17:52.299 [conn37] end connection 127.0.0.6:46140 (10 connections now open) m31000| Fri Jan 18 13:17:52.299 [initandlisten] connection accepted from 127.0.0.6:57863 #40 (11 connections now open) m31000| Fri Jan 18 13:17:52.323 [conn38] end connection 127.0.0.3:41334 (10 connections now open) m31000| Fri Jan 18 13:17:52.323 [initandlisten] connection accepted from 127.0.0.3:48607 #41 (11 connections now open) m31000| Fri Jan 18 13:17:52.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31000| Fri Jan 18 13:17:52.991 [conn39] end connection 127.0.0.4:39762 (10 connections now open) m31000| Fri Jan 18 13:17:52.991 [initandlisten] connection accepted from 127.0.0.4:42249 #42 (11 connections now open) m31003| Fri Jan 18 13:17:53.323 [rsSyncNotifier] repl: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:17:53.323 [conn1] command admin.$cmd command: { replSetGetStatus: true } ntoreturn:1 keyUpdates:0 reslen:1373 29064ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 83, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:22Z"), "pingMs" : 12, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 83, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 22, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 115, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "errmsg" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 83, "optime" : { "t" : 1358533044000, "i" : 752 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 111, "optime" : { "t" : 1358533044000, "i" : 2063 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 111, "optime" : { "t" : 1358533031000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:17:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 111, "optime" : { "t" : 1358533031000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:17:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 111, "optime" : { "t" : 1358532961000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 144, "optime" : { "t" : 1358533044000, "i" : 2063 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.5:31003", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:17:53.370 [FileAllocator] allocating new datafile /data/db/testReplSet-3/foo.ns, filling with zeroes... m31003| Fri Jan 18 13:17:53.519 [FileAllocator] done allocating datafile /data/db/testReplSet-3/foo.ns, size: 16MB, took 0.149 secs m31003| Fri Jan 18 13:17:53.519 [FileAllocator] allocating new datafile /data/db/testReplSet-3/foo.0, filling with zeroes... m31003| Fri Jan 18 13:17:53.669 [FileAllocator] done allocating datafile /data/db/testReplSet-3/foo.0, size: 16MB, took 0.149 secs m31003| Fri Jan 18 13:17:53.671 [repl writer worker 1] build index foo.bar { _id: 1 } m31003| Fri Jan 18 13:17:53.671 [repl writer worker 1] build index done. scanned 0 total records. 0 secs m31003| Fri Jan 18 13:17:54.323 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 m31000| Fri Jan 18 13:17:54.355 [conn25] end connection 127.0.0.5:54253 (10 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 151, "optime" : { "t" : 1358533078000, "i" : 4496 }, "optimeDate" : ISODate("2013-01-18T18:17:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 117, "optime" : { "t" : 1358533036000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:58Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 117, "optime" : { "t" : 1358533036000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 117, "optime" : { "t" : 1358533077000, "i" : 3196 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:58Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 116, "optime" : { "t" : 1358533078000, "i" : 3278 }, "optimeDate" : ISODate("2013-01-18T18:17:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 151, "optime" : { "t" : 1358533037000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:17:17Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 116, "optime" : { "t" : 1358533037000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:17:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 116, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 116, "optime" : { "t" : 1358533078000, "i" : 3275 }, "optimeDate" : ISODate("2013-01-18T18:17:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 116, "optime" : { "t" : 1358533076000, "i" : 5298 }, "optimeDate" : ISODate("2013-01-18T18:17:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 116, "optime" : { "t" : 1358533036000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:58Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 150, "optime" : { "t" : 1358533037000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:17:17Z"), "errmsg" : "syncing to: 127.0.0.2:31000", "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 116, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 116, "optime" : { "t" : 1358533076000, "i" : 5298 }, "optimeDate" : ISODate("2013-01-18T18:17:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:58Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 m31000| Fri Jan 18 13:18:02.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31004| Fri Jan 18 13:18:03.344 [conn18] end connection 127.0.0.2:53977 (4 connections now open) m31004| Fri Jan 18 13:18:03.344 [initandlisten] connection accepted from 127.0.0.2:52790 #22 (5 connections now open) m31003| Fri Jan 18 13:18:04.301 [conn19] end connection 127.0.0.6:48105 (3 connections now open) m31003| Fri Jan 18 13:18:04.301 [initandlisten] connection accepted from 127.0.0.6:33018 #22 (4 connections now open) m31004| Fri Jan 18 13:18:04.325 [conn19] end connection 127.0.0.3:40104 (4 connections now open) m31004| Fri Jan 18 13:18:04.326 [initandlisten] connection accepted from 127.0.0.3:57810 #23 (5 connections now open) m31004| Fri Jan 18 13:18:04.993 [conn20] end connection 127.0.0.4:47240 (4 connections now open) m31004| Fri Jan 18 13:18:04.993 [initandlisten] connection accepted from 127.0.0.4:59351 #24 (6 connections now open) m31002| Fri Jan 18 13:18:05.344 [conn22] end connection 127.0.0.2:37200 (4 connections now open) m31002| Fri Jan 18 13:18:05.344 [initandlisten] connection accepted from 127.0.0.2:44695 #26 (5 connections now open) m31001| Fri Jan 18 13:18:06.302 [conn22] end connection 127.0.0.6:35381 (4 connections now open) m31001| Fri Jan 18 13:18:06.302 [initandlisten] connection accepted from 127.0.0.6:44110 #26 (5 connections now open) m31002| Fri Jan 18 13:18:06.325 [conn23] end connection 127.0.0.3:49738 (4 connections now open) m31002| Fri Jan 18 13:18:06.325 [initandlisten] connection accepted from 127.0.0.3:41829 #27 (5 connections now open) m31003| Fri Jan 18 13:18:06.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Fri Jan 18 13:18:06.901 [conn23] end connection 127.0.0.5:48013 (4 connections now open) m31001| Fri Jan 18 13:18:06.913 [initandlisten] connection accepted from 127.0.0.5:51442 #27 (5 connections now open) m31001| Fri Jan 18 13:18:06.993 [conn24] end connection 127.0.0.4:47513 (4 connections now open) m31001| Fri Jan 18 13:18:06.993 [initandlisten] connection accepted from 127.0.0.4:39636 #28 (5 connections now open) m31003| Fri Jan 18 13:18:16.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Fri Jan 18 13:18:20.304 [conn24] end connection 127.0.0.6:54848 (4 connections now open) m31002| Fri Jan 18 13:18:20.304 [initandlisten] connection accepted from 127.0.0.6:44178 #28 (5 connections now open) m31003| Fri Jan 18 13:18:20.328 [conn20] end connection 127.0.0.3:55797 (3 connections now open) m31003| Fri Jan 18 13:18:20.328 [initandlisten] connection accepted from 127.0.0.3:34795 #23 (4 connections now open) m31003| Fri Jan 18 13:18:20.996 [conn21] end connection 127.0.0.4:43606 (3 connections now open) m31003| Fri Jan 18 13:18:20.996 [initandlisten] connection accepted from 127.0.0.4:49288 #24 (4 connections now open) m31002| Fri Jan 18 13:18:21.042 [conn25] end connection 127.0.0.5:35748 (4 connections now open) m31002| Fri Jan 18 13:18:21.120 [initandlisten] connection accepted from 127.0.0.5:58675 #29 (5 connections now open) m31001| Fri Jan 18 13:18:21.347 [conn25] end connection 127.0.0.2:58484 (4 connections now open) m31001| Fri Jan 18 13:18:21.347 [initandlisten] connection accepted from 127.0.0.2:52985 #29 (5 connections now open) m31003| Fri Jan 18 13:18:22.113 [rsMgr] replSet I don't see a primary and I can't elect myself m31000| Fri Jan 18 13:18:22.305 [conn40] end connection 127.0.0.6:57863 (9 connections now open) m31000| Fri Jan 18 13:18:22.305 [initandlisten] connection accepted from 127.0.0.6:49165 #43 (10 connections now open) m31000| Fri Jan 18 13:18:22.328 [conn41] end connection 127.0.0.3:48607 (9 connections now open) m31000| Fri Jan 18 13:18:22.328 [initandlisten] connection accepted from 127.0.0.3:46112 #44 (10 connections now open) m31000| Fri Jan 18 13:18:22.996 [conn42] end connection 127.0.0.4:42249 (9 connections now open) m31000| Fri Jan 18 13:18:22.996 [initandlisten] connection accepted from 127.0.0.4:36116 #45 (10 connections now open) m31003| Fri Jan 18 13:18:24.323 [rsSyncNotifier] repl: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:18:24.323 [conn1] command admin.$cmd command: { replSetGetStatus: true } ntoreturn:1 keyUpdates:0 reslen:1373 25861ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:17:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 117, "optime" : { "t" : 1358533036000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:58Z"), "pingMs" : 44, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 117, "optime" : { "t" : 1358533036000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:56Z"), "pingMs" : 18, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 149, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "errmsg" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 117, "optime" : { "t" : 1358533077000, "i" : 1062 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:17:58Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.2:31000" } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 142, "optime" : { "t" : 1358533078000, "i" : 4496 }, "optimeDate" : ISODate("2013-01-18T18:17:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 142, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 142, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 142, "optime" : { "t" : 1358532969000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 175, "optime" : { "t" : 1358533078000, "i" : 4496 }, "optimeDate" : ISODate("2013-01-18T18:17:58Z"), "self" : true } ], "ok" : 1 } m31000| Fri Jan 18 13:18:24.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.5:31003", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:18:24.393 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:24.393 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:24.393 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:24.393 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:18:24.393 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31001| Fri Jan 18 13:18:24.465 [initandlisten] connection accepted from 127.0.0.5:58615 #30 (6 connections now open) m31003| Fri Jan 18 13:18:24.583 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:24.583 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:24.583 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:24.583 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:24.609 [conn30] end connection 127.0.0.5:58615 (5 connections now open) m31004| Fri Jan 18 13:18:24.613 [conn21] end connection 127.0.0.5:33714 (4 connections now open) m31001| Fri Jan 18 13:18:24.748 [initandlisten] connection accepted from 127.0.0.5:60952 #31 (6 connections now open) m31003| Fri Jan 18 13:18:24.901 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:24.901 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:24.901 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:24.901 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:24.938 [conn31] end connection 127.0.0.5:60952 (5 connections now open) m31001| Fri Jan 18 13:18:24.938 [initandlisten] connection accepted from 127.0.0.5:50255 #32 (7 connections now open) m31004| Fri Jan 18 13:18:25.113 [initandlisten] connection accepted from 127.0.0.5:35143 #25 (5 connections now open) m31003| Fri Jan 18 13:18:25.122 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:25.122 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:25.122 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:25.122 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:25.156 [conn32] end connection 127.0.0.5:50255 (5 connections now open) m31001| Fri Jan 18 13:18:25.203 [initandlisten] connection accepted from 127.0.0.5:43582 #33 (6 connections now open) m31003| Fri Jan 18 13:18:25.352 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.3:31001 m31001| Fri Jan 18 13:18:25.352 [initandlisten] connection accepted from 127.0.0.5:40039 #34 (7 connections now open) m31001| Fri Jan 18 13:18:25.385 [conn33] end connection 127.0.0.5:43582 (6 connections now open) m31003| Fri Jan 18 13:18:25.389 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:25.389 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:25.389 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:25.389 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:25.484 [initandlisten] connection accepted from 127.0.0.5:51861 #35 (7 connections now open) m31001| Fri Jan 18 13:18:25.485 [rsGhostSync] handshake between 3 and 127.0.0.2:31000 m31000| Fri Jan 18 13:18:25.485 [initandlisten] connection accepted from 127.0.0.3:51746 #46 (11 connections now open) m31003| Fri Jan 18 13:18:25.582 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:25.582 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:25.582 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:25.582 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:25.618 [conn34] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533004000|301 } } cursorid:753629432870686 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:200405 nreturned:157538 reslen:2678166 133ms m31001| Fri Jan 18 13:18:25.674 [initandlisten] connection accepted from 127.0.0.5:38445 #36 (8 connections now open) m31001| Fri Jan 18 13:18:25.674 [conn35] end connection 127.0.0.5:51861 (7 connections now open) m31001| Fri Jan 18 13:18:25.789 [conn36] end connection 127.0.0.5:38445 (6 connections now open) m31003| Fri Jan 18 13:18:25.793 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:25.793 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:25.793 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:25.793 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:25.804 [initandlisten] connection accepted from 127.0.0.5:36384 #37 (7 connections now open) m31003| Fri Jan 18 13:18:25.911 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:25.911 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:25.911 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:25.911 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:25.947 [initandlisten] connection accepted from 127.0.0.5:60059 #38 (8 connections now open) m31001| Fri Jan 18 13:18:25.950 [conn37] end connection 127.0.0.5:36384 (7 connections now open) m31003| Fri Jan 18 13:18:26.053 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:26.053 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:26.053 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:26.053 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:26.110 [conn38] end connection 127.0.0.5:60059 (6 connections now open) m31001| Fri Jan 18 13:18:26.110 [initandlisten] connection accepted from 127.0.0.5:42928 #39 (8 connections now open) m31001| Fri Jan 18 13:18:26.334 [conn39] end connection 127.0.0.5:42928 (6 connections now open) m31003| Fri Jan 18 13:18:26.463 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:26.463 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:26.463 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:26.463 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:26.485 [slaveTracking] build index local.slaves { _id: 1 } m31001| Fri Jan 18 13:18:26.486 [slaveTracking] build index done. scanned 0 total records. 0 secs m31001| Fri Jan 18 13:18:26.594 [initandlisten] connection accepted from 127.0.0.5:40771 #40 (7 connections now open) m31003| Fri Jan 18 13:18:26.724 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:26.724 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:26.724 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:26.724 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:26.776 [conn40] end connection 127.0.0.5:40771 (6 connections now open) m31001| Fri Jan 18 13:18:26.776 [initandlisten] connection accepted from 127.0.0.5:33344 #41 (8 connections now open) m31003| Fri Jan 18 13:18:26.915 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:26.915 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:26.915 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:26.915 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:27.007 [conn41] end connection 127.0.0.5:33344 (6 connections now open) m31001| Fri Jan 18 13:18:27.029 [initandlisten] connection accepted from 127.0.0.5:55551 #42 (7 connections now open) m31001| Fri Jan 18 13:18:27.082 [conn42] end connection 127.0.0.5:55551 (6 connections now open) m31003| Fri Jan 18 13:18:27.086 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:27.086 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:27.086 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:27.086 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:27.086 [initandlisten] connection accepted from 127.0.0.5:54034 #43 (7 connections now open) m31003| Fri Jan 18 13:18:27.289 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:27.289 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:27.289 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:27.289 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:27.377 [initandlisten] connection accepted from 127.0.0.5:55123 #44 (8 connections now open) m31001| Fri Jan 18 13:18:27.377 [conn43] end connection 127.0.0.5:54034 (7 connections now open) m31003| Fri Jan 18 13:18:27.455 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:27.455 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:27.455 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:27.455 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:27.534 [conn44] end connection 127.0.0.5:55123 (6 connections now open) m31001| Fri Jan 18 13:18:27.587 [initandlisten] connection accepted from 127.0.0.5:40378 #45 (7 connections now open) m31001| Fri Jan 18 13:18:27.696 [conn45] end connection 127.0.0.5:40378 (6 connections now open) m31003| Fri Jan 18 13:18:27.698 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:27.698 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:27.698 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:27.698 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:27.735 [initandlisten] connection accepted from 127.0.0.5:46936 #46 (7 connections now open) m31003| Fri Jan 18 13:18:27.838 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:27.838 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:27.838 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:27.838 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:27.898 [conn46] end connection 127.0.0.5:46936 (6 connections now open) m31001| Fri Jan 18 13:18:27.924 [initandlisten] connection accepted from 127.0.0.5:37518 #47 (7 connections now open) m31003| Fri Jan 18 13:18:28.055 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:28.055 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:28.055 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:28.055 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:28.150 [initandlisten] connection accepted from 127.0.0.5:44197 #48 (8 connections now open) m31001| Fri Jan 18 13:18:28.150 [conn47] end connection 127.0.0.5:37518 (6 connections now open) m31003| Fri Jan 18 13:18:28.275 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:28.275 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:28.275 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:28.275 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:28.286 [conn48] end connection 127.0.0.5:44197 (6 connections now open) m31001| Fri Jan 18 13:18:28.360 [initandlisten] connection accepted from 127.0.0.5:56672 #49 (7 connections now open) m31003| Fri Jan 18 13:18:28.419 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:28.419 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:28.419 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:28.419 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:28.473 [conn49] end connection 127.0.0.5:56672 (6 connections now open) m31001| Fri Jan 18 13:18:28.543 [initandlisten] connection accepted from 127.0.0.5:38449 #50 (7 connections now open) m31001| Fri Jan 18 13:18:28.616 [conn50] end connection 127.0.0.5:38449 (6 connections now open) m31003| Fri Jan 18 13:18:28.618 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:28.618 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:28.618 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:28.618 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31000| Fri Jan 18 13:18:28.696 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:244321 122ms m31001| Fri Jan 18 13:18:28.720 [initandlisten] connection accepted from 127.0.0.5:42883 #51 (7 connections now open) m31003| Fri Jan 18 13:18:28.802 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:28.802 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:28.802 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:28.802 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:28.865 [initandlisten] connection accepted from 127.0.0.5:37410 #52 (8 connections now open) m31001| Fri Jan 18 13:18:28.865 [conn51] end connection 127.0.0.5:42883 (7 connections now open) m31003| Fri Jan 18 13:18:28.948 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:28.948 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:28.948 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:28.948 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:28.963 [conn52] end connection 127.0.0.5:37410 (6 connections now open) m31001| Fri Jan 18 13:18:28.990 [initandlisten] connection accepted from 127.0.0.5:56200 #53 (7 connections now open) m31003| Fri Jan 18 13:18:29.108 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:29.108 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:29.108 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:29.108 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:29.184 [initandlisten] connection accepted from 127.0.0.5:48736 #54 (8 connections now open) m31001| Fri Jan 18 13:18:29.184 [conn53] end connection 127.0.0.5:56200 (7 connections now open) m31003| Fri Jan 18 13:18:29.270 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:29.270 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:29.270 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:18:29.270 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31001| Fri Jan 18 13:18:29.313 [conn54] end connection 127.0.0.5:48736 (6 connections now open) m31002| Fri Jan 18 13:18:29.359 [initandlisten] connection accepted from 127.0.0.5:54022 #30 (6 connections now open) m31003| Fri Jan 18 13:18:29.496 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 m31003| Fri Jan 18 13:18:29.496 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:29.496 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:29.496 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:29.584 [conn30] end connection 127.0.0.5:54022 (5 connections now open) m31001| Fri Jan 18 13:18:29.595 [conn34] end connection 127.0.0.5:40039 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31002| Fri Jan 18 13:18:29.615 [initandlisten] connection accepted from 127.0.0.5:43214 #31 (6 connections now open) m31002| Fri Jan 18 13:18:29.615 [initandlisten] connection accepted from 127.0.0.5:35503 #32 (7 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 182, "optime" : { "t" : 1358533109000, "i" : 5470 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 148, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 148, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 148, "optime" : { "t" : 1358533109000, "i" : 3384 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 147, "optime" : { "t" : 1358533108000, "i" : 3084 }, "optimeDate" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 182, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:28Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358533004000, "i" : 1801 }, "optimeDate" : ISODate("2013-01-18T18:16:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:29Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358533108000, "i" : 3032 }, "optimeDate" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 147, "optime" : { "t" : 1358533108000, "i" : 8926 }, "optimeDate" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 181, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358533004000, "i" : 2201 }, "optimeDate" : ISODate("2013-01-18T18:16:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:29Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358533108000, "i" : 8912 }, "optimeDate" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:29Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 148, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 24 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 148, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 19 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358533004000, "i" : 2501 }, "optimeDate" : ISODate("2013-01-18T18:16:44Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 148, "optime" : { "t" : 1358533107000, "i" : 2494 }, "optimeDate" : ISODate("2013-01-18T18:18:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:29.699 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:29.699 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:29.699 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:29.699 [conn31] end connection 127.0.0.5:43214 (6 connections now open) m31002| Fri Jan 18 13:18:29.699 [rsGhostSync] handshake between 3 and 127.0.0.2:31000 m31000| Fri Jan 18 13:18:29.699 [initandlisten] connection accepted from 127.0.0.4:41546 #47 (12 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 147, "optime" : { "t" : 1358533108000, "i" : 2734 }, "optimeDate" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:28Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:28Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358533004000, "i" : 1801 }, "optimeDate" : ISODate("2013-01-18T18:16:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358533109000, "i" : 5470 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31002| Fri Jan 18 13:18:29.779 [initandlisten] connection accepted from 127.0.0.5:57716 #33 (7 connections now open) m31002| Fri Jan 18 13:18:29.836 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533004000|2501 } } cursorid:768915420396460 ntoreturn:0 keyUpdates:0 locks(micros) r:137493 nreturned:155338 reslen:2640766 137ms m31003| Fri Jan 18 13:18:29.927 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:29.927 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:29.927 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:29.959 [conn33] end connection 127.0.0.5:57716 (6 connections now open) m31002| Fri Jan 18 13:18:29.989 [initandlisten] connection accepted from 127.0.0.5:37070 #34 (7 connections now open) m31003| Fri Jan 18 13:18:30.155 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:30.155 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:30.155 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:30.167 [initandlisten] connection accepted from 127.0.0.5:52339 #35 (8 connections now open) m31002| Fri Jan 18 13:18:30.233 [conn34] end connection 127.0.0.5:37070 (7 connections now open) m31003| Fri Jan 18 13:18:30.300 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:30.300 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:30.300 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:30.312 [initandlisten] connection accepted from 127.0.0.5:57801 #36 (8 connections now open) m31002| Fri Jan 18 13:18:30.324 [conn35] end connection 127.0.0.5:52339 (7 connections now open) m31003| Fri Jan 18 13:18:30.569 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:30.569 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:30.569 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:30.620 [conn36] end connection 127.0.0.5:57801 (6 connections now open) m31002| Fri Jan 18 13:18:30.700 [slaveTracking] build index local.slaves { _id: 1 } m31002| Fri Jan 18 13:18:30.701 [slaveTracking] build index done. scanned 0 total records. 0 secs m31002| Fri Jan 18 13:18:30.701 [initandlisten] connection accepted from 127.0.0.5:38971 #37 (7 connections now open) m31002| Fri Jan 18 13:18:30.751 [conn37] end connection 127.0.0.5:38971 (6 connections now open) m31003| Fri Jan 18 13:18:30.755 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:30.755 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:30.755 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:30.833 [initandlisten] connection accepted from 127.0.0.5:34313 #38 (7 connections now open) m31003| Fri Jan 18 13:18:30.963 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:30.963 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:30.963 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:31.014 [conn38] end connection 127.0.0.5:34313 (6 connections now open) m31002| Fri Jan 18 13:18:31.014 [initandlisten] connection accepted from 127.0.0.5:53442 #39 (8 connections now open) m31002| Fri Jan 18 13:18:31.212 [conn39] end connection 127.0.0.5:53442 (6 connections now open) m31003| Fri Jan 18 13:18:31.213 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:31.213 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:31.213 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:31.213 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:18:31.213 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31001| Fri Jan 18 13:18:31.246 [initandlisten] connection accepted from 127.0.0.5:47748 #55 (6 connections now open) m31001| Fri Jan 18 13:18:31.343 [conn55] end connection 127.0.0.5:47748 (5 connections now open) m31003| Fri Jan 18 13:18:31.346 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.3:31001 m31002| Fri Jan 18 13:18:31.402 [conn32] end connection 127.0.0.5:35503 (5 connections now open) m31001| Fri Jan 18 13:18:31.424 [initandlisten] connection accepted from 127.0.0.5:40223 #56 (6 connections now open) m31003| Fri Jan 18 13:18:31.424 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:31.424 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:31.424 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:31.424 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:31.496 [initandlisten] connection accepted from 127.0.0.5:50313 #57 (7 connections now open) m31003| Fri Jan 18 13:18:31.598 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:31.598 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:31.598 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:31.598 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:31.640 [conn57] end connection 127.0.0.5:50313 (6 connections now open) m31001| Fri Jan 18 13:18:31.640 [initandlisten] connection accepted from 127.0.0.5:41220 #58 (8 connections now open) m31001| Fri Jan 18 13:18:31.758 [conn58] end connection 127.0.0.5:41220 (6 connections now open) m31003| Fri Jan 18 13:18:31.758 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:31.758 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:31.758 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:31.758 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:31.768 [conn56] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533005000|823 } } cursorid:779533028279232 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:225098 nreturned:154438 reslen:2625466 128ms m31001| Fri Jan 18 13:18:31.827 [initandlisten] connection accepted from 127.0.0.5:52814 #59 (7 connections now open) m31000| Fri Jan 18 13:18:31.842 [conn46] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533004000|401 } } cursorid:756800973567745 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:256265 nreturned:199500 reslen:3391520 202ms m31003| Fri Jan 18 13:18:31.957 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:31.957 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:31.957 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:31.957 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:31.959 [conn59] end connection 127.0.0.5:52814 (6 connections now open) m31001| Fri Jan 18 13:18:32.037 [initandlisten] connection accepted from 127.0.0.5:49138 #60 (7 connections now open) m31003| Fri Jan 18 13:18:32.109 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:32.109 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:32.109 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:32.109 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:32.184 [conn60] end connection 127.0.0.5:49138 (6 connections now open) m31001| Fri Jan 18 13:18:32.196 [initandlisten] connection accepted from 127.0.0.5:50724 #61 (7 connections now open) m31003| Fri Jan 18 13:18:32.412 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:32.412 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:32.412 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:32.412 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:32.416 [conn61] end connection 127.0.0.5:50724 (6 connections now open) m31001| Fri Jan 18 13:18:32.556 [initandlisten] connection accepted from 127.0.0.5:46197 #62 (7 connections now open) m31003| Fri Jan 18 13:18:32.655 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:32.655 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:32.655 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:32.655 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:32.675 [initandlisten] connection accepted from 127.0.0.5:53450 #63 (8 connections now open) m31001| Fri Jan 18 13:18:32.675 [conn62] end connection 127.0.0.5:46197 (6 connections now open) m31003| Fri Jan 18 13:18:32.796 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:32.796 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:32.796 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:32.796 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:32.891 [initandlisten] connection accepted from 127.0.0.5:41355 #64 (8 connections now open) m31001| Fri Jan 18 13:18:32.891 [conn63] end connection 127.0.0.5:53450 (6 connections now open) m31003| Fri Jan 18 13:18:33.037 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:33.037 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:33.037 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:33.037 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:33.094 [initandlisten] connection accepted from 127.0.0.5:51406 #65 (8 connections now open) m31001| Fri Jan 18 13:18:33.094 [conn64] end connection 127.0.0.5:41355 (6 connections now open) m31001| Fri Jan 18 13:18:33.233 [conn65] end connection 127.0.0.5:51406 (6 connections now open) m31003| Fri Jan 18 13:18:33.235 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:33.235 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:33.235 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:33.235 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:33.316 [initandlisten] connection accepted from 127.0.0.5:40313 #66 (7 connections now open) m31004| Fri Jan 18 13:18:33.348 [conn22] end connection 127.0.0.2:52790 (4 connections now open) m31004| Fri Jan 18 13:18:33.348 [initandlisten] connection accepted from 127.0.0.2:56902 #26 (5 connections now open) m31003| Fri Jan 18 13:18:33.371 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:33.371 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:33.371 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:33.371 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:33.409 [initandlisten] connection accepted from 127.0.0.5:34770 #67 (8 connections now open) m31001| Fri Jan 18 13:18:33.409 [conn66] end connection 127.0.0.5:40313 (6 connections now open) m31003| Fri Jan 18 13:18:33.592 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:33.592 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:33.592 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:33.592 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:33.673 [initandlisten] connection accepted from 127.0.0.5:58793 #68 (8 connections now open) m31001| Fri Jan 18 13:18:33.673 [conn67] end connection 127.0.0.5:34770 (7 connections now open) m31003| Fri Jan 18 13:18:33.765 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:33.765 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:33.765 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:33.765 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:33.832 [initandlisten] connection accepted from 127.0.0.5:44615 #69 (8 connections now open) m31001| Fri Jan 18 13:18:33.832 [conn68] end connection 127.0.0.5:58793 (6 connections now open) m31003| Fri Jan 18 13:18:34.031 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:34.031 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:34.031 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:34.031 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:34.125 [conn69] end connection 127.0.0.5:44615 (6 connections now open) m31001| Fri Jan 18 13:18:34.127 [initandlisten] connection accepted from 127.0.0.5:56357 #70 (7 connections now open) m31001| Fri Jan 18 13:18:34.247 [conn70] end connection 127.0.0.5:56357 (6 connections now open) m31003| Fri Jan 18 13:18:34.249 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:34.249 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:34.249 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:34.249 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:34.283 [initandlisten] connection accepted from 127.0.0.5:55631 #71 (7 connections now open) m31003| Fri Jan 18 13:18:34.307 [conn22] end connection 127.0.0.6:33018 (3 connections now open) m31003| Fri Jan 18 13:18:34.307 [initandlisten] connection accepted from 127.0.0.6:58508 #25 (4 connections now open) m31004| Fri Jan 18 13:18:34.330 [conn23] end connection 127.0.0.3:57810 (4 connections now open) m31004| Fri Jan 18 13:18:34.330 [initandlisten] connection accepted from 127.0.0.3:49876 #27 (5 connections now open) m31000| Fri Jan 18 13:18:34.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:18:34.413 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:34.413 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:34.413 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:34.413 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:34.434 [conn71] end connection 127.0.0.5:55631 (6 connections now open) m31001| Fri Jan 18 13:18:34.464 [initandlisten] connection accepted from 127.0.0.5:45030 #72 (7 connections now open) m31003| Fri Jan 18 13:18:34.557 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:34.557 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:34.557 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:34.557 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:34.591 [conn72] end connection 127.0.0.5:45030 (6 connections now open) m31001| Fri Jan 18 13:18:34.631 [initandlisten] connection accepted from 127.0.0.5:42857 #73 (7 connections now open) m31001| Fri Jan 18 13:18:34.791 [conn73] end connection 127.0.0.5:42857 (6 connections now open) m31003| Fri Jan 18 13:18:34.793 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:34.793 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:34.793 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:34.793 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:34.881 [initandlisten] connection accepted from 127.0.0.5:39463 #74 (7 connections now open) m31003| Fri Jan 18 13:18:34.973 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:34.973 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:34.973 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:34.973 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31004| Fri Jan 18 13:18:34.998 [conn24] end connection 127.0.0.4:59351 (4 connections now open) m31004| Fri Jan 18 13:18:34.998 [initandlisten] connection accepted from 127.0.0.4:46753 #28 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 188, "optime" : { "t" : 1358533115000, "i" : 94 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 154, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:34Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 154, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:34Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 154, "optime" : { "t" : 1358533113000, "i" : 3195 }, "optimeDate" : ISODate("2013-01-18T18:18:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 153, "optime" : { "t" : 1358533114000, "i" : 2926 }, "optimeDate" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 188, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 153, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:34Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 153, "optime" : { "t" : 1358533005000, "i" : 2223 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 153, "optime" : { "t" : 1358533114000, "i" : 2912 }, "optimeDate" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 153, "optime" : { "t" : 1358533114000, "i" : 8665 }, "optimeDate" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 153, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 187, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 153, "optime" : { "t" : 1358533005000, "i" : 2623 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 153, "optime" : { "t" : 1358533114000, "i" : 8658 }, "optimeDate" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:35Z"), "myState" : 2, "syncingTo" : "127.0.0.3:31001", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 154, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:34Z"), "pingMs" : 15 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 154, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:34Z"), "pingMs" : 23 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 186, "optime" : { "t" : 1358533005000, "i" : 2623 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "errmsg" : "syncing to: 127.0.0.3:31001", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 154, "optime" : { "t" : 1358533109000, "i" : 5470 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 153, "optime" : { "t" : 1358533114000, "i" : 2762 }, "optimeDate" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 153, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:34Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 153, "optime" : { "t" : 1358533044000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:17:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:34Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 153, "optime" : { "t" : 1358533005000, "i" : 2223 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 186, "optime" : { "t" : 1358533115000, "i" : 89 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.3:31001" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:18:35.067 [conn74] end connection 127.0.0.5:39463 (6 connections now open) m31001| Fri Jan 18 13:18:35.145 [initandlisten] connection accepted from 127.0.0.5:43707 #75 (7 connections now open) m31003| Fri Jan 18 13:18:35.204 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:35.204 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:35.204 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:35.204 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:35.236 [conn75] end connection 127.0.0.5:43707 (6 connections now open) m31001| Fri Jan 18 13:18:35.236 [initandlisten] connection accepted from 127.0.0.5:45689 #76 (8 connections now open) m31002| Fri Jan 18 13:18:35.349 [conn26] end connection 127.0.0.2:44695 (4 connections now open) m31002| Fri Jan 18 13:18:35.349 [initandlisten] connection accepted from 127.0.0.2:52133 #40 (5 connections now open) m31003| Fri Jan 18 13:18:35.458 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:35.458 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:35.458 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:35.458 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:35.537 [initandlisten] connection accepted from 127.0.0.5:55781 #77 (8 connections now open) m31001| Fri Jan 18 13:18:35.537 [conn76] end connection 127.0.0.5:45689 (6 connections now open) m31003| Fri Jan 18 13:18:35.617 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:35.617 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:35.617 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:35.617 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:35.632 [initandlisten] connection accepted from 127.0.0.5:59665 #78 (8 connections now open) m31001| Fri Jan 18 13:18:35.633 [conn77] end connection 127.0.0.5:55781 (7 connections now open) m31001| Fri Jan 18 13:18:35.720 [conn78] end connection 127.0.0.5:59665 (6 connections now open) m31003| Fri Jan 18 13:18:35.724 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:35.724 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:35.724 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:35.724 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:35.724 [initandlisten] connection accepted from 127.0.0.5:43059 #79 (7 connections now open) m31003| Fri Jan 18 13:18:35.868 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:35.868 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:35.868 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:35.868 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:35.898 [conn79] end connection 127.0.0.5:43059 (6 connections now open) m31001| Fri Jan 18 13:18:35.950 [initandlisten] connection accepted from 127.0.0.5:60703 #80 (7 connections now open) m31003| Fri Jan 18 13:18:36.095 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:36.095 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:36.095 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:36.095 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:36.132 [conn80] end connection 127.0.0.5:60703 (6 connections now open) m31001| Fri Jan 18 13:18:36.184 [initandlisten] connection accepted from 127.0.0.5:45871 #81 (7 connections now open) m31003| Fri Jan 18 13:18:36.289 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:36.289 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:36.289 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:36.289 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:36.301 [conn81] end connection 127.0.0.5:45871 (6 connections now open) m31001| Fri Jan 18 13:18:36.307 [conn26] end connection 127.0.0.6:44110 (5 connections now open) m31001| Fri Jan 18 13:18:36.307 [initandlisten] connection accepted from 127.0.0.6:52681 #82 (6 connections now open) m31001| Fri Jan 18 13:18:36.322 [initandlisten] connection accepted from 127.0.0.5:60800 #83 (7 connections now open) m31002| Fri Jan 18 13:18:36.330 [conn27] end connection 127.0.0.3:41829 (4 connections now open) m31002| Fri Jan 18 13:18:36.331 [initandlisten] connection accepted from 127.0.0.3:55347 #41 (6 connections now open) m31003| Fri Jan 18 13:18:36.371 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:36.371 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:36.371 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:36.371 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:36.428 [conn83] end connection 127.0.0.5:60800 (6 connections now open) m31001| Fri Jan 18 13:18:36.468 [initandlisten] connection accepted from 127.0.0.5:53250 #84 (7 connections now open) m31001| Fri Jan 18 13:18:36.539 [conn84] end connection 127.0.0.5:53250 (6 connections now open) m31003| Fri Jan 18 13:18:36.543 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:36.543 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:36.543 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:36.543 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:36.568 [initandlisten] connection accepted from 127.0.0.5:53734 #85 (7 connections now open) m31003| Fri Jan 18 13:18:36.688 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:36.688 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:36.688 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:36.688 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:36.756 [conn85] end connection 127.0.0.5:53734 (6 connections now open) m31001| Fri Jan 18 13:18:36.842 [initandlisten] connection accepted from 127.0.0.5:46150 #86 (7 connections now open) m31001| Fri Jan 18 13:18:36.998 [conn28] end connection 127.0.0.4:39636 (6 connections now open) m31001| Fri Jan 18 13:18:36.998 [initandlisten] connection accepted from 127.0.0.4:58333 #87 (7 connections now open) m31001| Fri Jan 18 13:18:37.022 [conn86] end connection 127.0.0.5:46150 (6 connections now open) m31003| Fri Jan 18 13:18:37.026 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:37.026 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:37.026 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:37.026 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:37.138 [initandlisten] connection accepted from 127.0.0.5:39890 #88 (7 connections now open) m31001| Fri Jan 18 13:18:37.226 [initandlisten] connection accepted from 127.0.0.5:55319 #89 (8 connections now open) m31001| Fri Jan 18 13:18:37.268 [conn27] end connection 127.0.0.5:51442 (7 connections now open) m31003| Fri Jan 18 13:18:37.283 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:37.283 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:37.283 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:37.283 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:37.365 [conn88] end connection 127.0.0.5:39890 (6 connections now open) m31001| Fri Jan 18 13:18:37.365 [initandlisten] connection accepted from 127.0.0.5:52669 #90 (7 connections now open) m31003| Fri Jan 18 13:18:37.418 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:37.418 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:37.418 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:37.418 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:37.423 [conn90] end connection 127.0.0.5:52669 (6 connections now open) m31001| Fri Jan 18 13:18:37.508 [initandlisten] connection accepted from 127.0.0.5:40857 #91 (7 connections now open) m31003| Fri Jan 18 13:18:37.655 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:37.655 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:37.655 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:37.655 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:37.689 [conn91] end connection 127.0.0.5:40857 (6 connections now open) m31001| Fri Jan 18 13:18:37.691 [initandlisten] connection accepted from 127.0.0.5:40916 #92 (7 connections now open) m31003| Fri Jan 18 13:18:37.814 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:37.814 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:37.814 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:37.814 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:37.832 [conn92] end connection 127.0.0.5:40916 (6 connections now open) m31001| Fri Jan 18 13:18:37.911 [initandlisten] connection accepted from 127.0.0.5:42234 #93 (7 connections now open) m31001| Fri Jan 18 13:18:38.027 [conn93] end connection 127.0.0.5:42234 (6 connections now open) m31003| Fri Jan 18 13:18:38.037 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:38.037 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:38.037 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:38.037 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:38.099 [initandlisten] connection accepted from 127.0.0.5:46013 #94 (7 connections now open) m31003| Fri Jan 18 13:18:38.320 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:38.320 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:38.320 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:38.320 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:38.338 [conn94] end connection 127.0.0.5:46013 (6 connections now open) m31001| Fri Jan 18 13:18:38.460 [initandlisten] connection accepted from 127.0.0.5:55440 #95 (7 connections now open) m31001| Fri Jan 18 13:18:38.591 [conn95] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533005000|4323 } } cursorid:809515527221594 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 639 locks(micros) r:102307 nreturned:101 reslen:10423 124ms m31003| Fri Jan 18 13:18:38.593 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:38.593 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:38.593 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:38.593 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:18:38.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Fri Jan 18 13:18:38.679 [initandlisten] connection accepted from 127.0.0.5:47717 #96 (8 connections now open) m31001| Fri Jan 18 13:18:38.679 [conn95] end connection 127.0.0.5:55440 (7 connections now open) m31003| Fri Jan 18 13:18:38.731 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:38.731 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:38.731 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:38.731 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:38.795 [initandlisten] connection accepted from 127.0.0.5:55297 #97 (8 connections now open) m31001| Fri Jan 18 13:18:38.795 [conn96] end connection 127.0.0.5:47717 (7 connections now open) m31003| Fri Jan 18 13:18:38.905 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:38.905 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:38.905 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:38.905 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:38.950 [initandlisten] connection accepted from 127.0.0.5:44608 #98 (8 connections now open) m31001| Fri Jan 18 13:18:38.950 [conn97] end connection 127.0.0.5:55297 (6 connections now open) m31003| Fri Jan 18 13:18:39.081 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:39.081 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:39.081 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:39.081 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:39.114 [conn98] end connection 127.0.0.5:44608 (6 connections now open) m31001| Fri Jan 18 13:18:39.114 [initandlisten] connection accepted from 127.0.0.5:59712 #99 (8 connections now open) m31003| Fri Jan 18 13:18:39.256 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:39.256 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:39.256 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:39.256 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:39.290 [initandlisten] connection accepted from 127.0.0.5:34557 #100 (8 connections now open) m31001| Fri Jan 18 13:18:39.323 [conn99] end connection 127.0.0.5:59712 (7 connections now open) m31003| Fri Jan 18 13:18:39.436 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:39.436 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:39.436 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:39.436 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:39.489 [initandlisten] connection accepted from 127.0.0.5:47342 #101 (8 connections now open) m31001| Fri Jan 18 13:18:39.489 [conn100] end connection 127.0.0.5:34557 (7 connections now open) m31003| Fri Jan 18 13:18:39.619 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:39.619 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:39.619 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:39.619 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:39.645 [initandlisten] connection accepted from 127.0.0.5:43921 #102 (8 connections now open) m31001| Fri Jan 18 13:18:39.645 [conn101] end connection 127.0.0.5:47342 (6 connections now open) m31003| Fri Jan 18 13:18:39.765 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:39.765 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:39.765 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:39.765 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:39.825 [conn102] end connection 127.0.0.5:43921 (6 connections now open) m31001| Fri Jan 18 13:18:39.825 [initandlisten] connection accepted from 127.0.0.5:41462 #103 (8 connections now open) m31003| Fri Jan 18 13:18:39.942 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:39.942 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:39.942 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:39.942 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:40.018 [conn103] end connection 127.0.0.5:41462 (6 connections now open) m31001| Fri Jan 18 13:18:40.058 [initandlisten] connection accepted from 127.0.0.5:39858 #104 (7 connections now open) m31003| Fri Jan 18 13:18:40.172 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:40.172 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:40.172 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:40.172 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:40.197 [conn104] end connection 127.0.0.5:39858 (6 connections now open) m31001| Fri Jan 18 13:18:40.280 [initandlisten] connection accepted from 127.0.0.5:58541 #105 (7 connections now open) STATUS FROM : 127.0.0.2:31000 m31003| Fri Jan 18 13:18:40.387 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:40.387 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:40.387 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:40.387 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 193, "optime" : { "t" : 1358533120000, "i" : 3246 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 159, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 159, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 159, "optime" : { "t" : 1358533119000, "i" : 2941 }, "optimeDate" : ISODate("2013-01-18T18:18:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 158, "optime" : { "t" : 1358533120000, "i" : 2758 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 193, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 158, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 158, "optime" : { "t" : 1358533005000, "i" : 5323 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 158, "optime" : { "t" : 1358533120000, "i" : 2754 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 158, "optime" : { "t" : 1358533118000, "i" : 8162 }, "optimeDate" : ISODate("2013-01-18T18:18:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 158, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 192, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 158, "optime" : { "t" : 1358533005000, "i" : 4623 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:39Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 158, "optime" : { "t" : 1358533118000, "i" : 8143 }, "optimeDate" : ISODate("2013-01-18T18:18:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:40Z"), "myState" : 2, "syncingTo" : "127.0.0.3:31001", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 159, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:40Z"), "pingMs" : 18 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 159, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 19 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 191, "optime" : { "t" : 1358533005000, "i" : 5423 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "errmsg" : "syncing to: 127.0.0.3:31001", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 159, "optime" : { "t" : 1358533120000, "i" : 1044 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 158, "optime" : { "t" : 1358533120000, "i" : 2630 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 158, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 158, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 158, "optime" : { "t" : 1358533005000, "i" : 5323 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 191, "optime" : { "t" : 1358533120000, "i" : 3246 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.3:31001" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:18:40.431 [initandlisten] connection accepted from 127.0.0.5:40535 #106 (8 connections now open) m31001| Fri Jan 18 13:18:40.431 [conn105] end connection 127.0.0.5:58541 (6 connections now open) m31003| Fri Jan 18 13:18:40.639 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:40.639 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:40.639 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:40.639 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:40.725 [initandlisten] connection accepted from 127.0.0.5:36200 #107 (8 connections now open) m31001| Fri Jan 18 13:18:40.725 [conn106] end connection 127.0.0.5:40535 (6 connections now open) m31001| Fri Jan 18 13:18:40.920 [conn107] end connection 127.0.0.5:36200 (6 connections now open) m31003| Fri Jan 18 13:18:40.924 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:40.924 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:40.924 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:40.924 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:40.926 [initandlisten] connection accepted from 127.0.0.5:44728 #108 (7 connections now open) m31001| Fri Jan 18 13:18:41.030 [conn108] end connection 127.0.0.5:44728 (6 connections now open) m31003| Fri Jan 18 13:18:41.060 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:41.060 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:41.060 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:41.060 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:41.167 [initandlisten] connection accepted from 127.0.0.5:45797 #109 (7 connections now open) m31001| Fri Jan 18 13:18:41.234 [conn109] end connection 127.0.0.5:45797 (6 connections now open) m31003| Fri Jan 18 13:18:41.238 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:41.238 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:41.238 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:41.238 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:41.358 [initandlisten] connection accepted from 127.0.0.5:56321 #110 (7 connections now open) m31001| Fri Jan 18 13:18:41.435 [conn110] end connection 127.0.0.5:56321 (6 connections now open) m31003| Fri Jan 18 13:18:41.439 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:41.439 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:41.439 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:18:41.439 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31002| Fri Jan 18 13:18:41.489 [initandlisten] connection accepted from 127.0.0.5:57897 #42 (6 connections now open) m31003| Fri Jan 18 13:18:41.600 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:41.600 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:41.600 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 m31003| Fri Jan 18 13:18:41.600 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:41.609 [conn42] end connection 127.0.0.5:57897 (5 connections now open) m31001| Fri Jan 18 13:18:41.645 [conn56] end connection 127.0.0.5:40223 (5 connections now open) m31002| Fri Jan 18 13:18:41.670 [initandlisten] connection accepted from 127.0.0.5:42989 #43 (6 connections now open) m31002| Fri Jan 18 13:18:41.759 [initandlisten] connection accepted from 127.0.0.5:32801 #44 (7 connections now open) m31003| Fri Jan 18 13:18:41.876 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:41.876 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:41.876 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:41.974 [initandlisten] connection accepted from 127.0.0.5:58442 #45 (8 connections now open) m31002| Fri Jan 18 13:18:41.974 [conn43] end connection 127.0.0.5:42989 (6 connections now open) m31002| Fri Jan 18 13:18:42.123 [conn44] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533005000|6023 } } cursorid:820819931306255 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:296780 nreturned:166750 reslen:2834770 149ms m31000| Fri Jan 18 13:18:42.171 [conn47] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533005000|23 } } cursorid:774749022007621 ntoreturn:0 keyUpdates:0 locks(micros) r:196750 nreturned:237300 reslen:4034120 196ms m31003| Fri Jan 18 13:18:42.181 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:42.181 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:42.181 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:42.263 [initandlisten] connection accepted from 127.0.0.5:54735 #46 (8 connections now open) m31002| Fri Jan 18 13:18:42.276 [conn45] end connection 127.0.0.5:58442 (7 connections now open) m31003| Fri Jan 18 13:18:42.328 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:42.328 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:42.328 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:42.341 [initandlisten] connection accepted from 127.0.0.5:34849 #47 (8 connections now open) m31002| Fri Jan 18 13:18:42.341 [conn46] end connection 127.0.0.5:54735 (7 connections now open) m31003| Fri Jan 18 13:18:42.495 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:42.495 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:42.495 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:42.535 [initandlisten] connection accepted from 127.0.0.5:38642 #48 (8 connections now open) m31002| Fri Jan 18 13:18:42.583 [conn47] end connection 127.0.0.5:34849 (7 connections now open) m31003| Fri Jan 18 13:18:42.626 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:42.626 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:42.626 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:42.636 [conn48] end connection 127.0.0.5:38642 (6 connections now open) m31002| Fri Jan 18 13:18:42.655 [initandlisten] connection accepted from 127.0.0.5:35833 #49 (7 connections now open) m31003| Fri Jan 18 13:18:42.829 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:42.829 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:42.829 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:42.840 [conn49] end connection 127.0.0.5:35833 (6 connections now open) m31002| Fri Jan 18 13:18:42.925 [initandlisten] connection accepted from 127.0.0.5:38817 #50 (7 connections now open) m31002| Fri Jan 18 13:18:43.089 [conn50] end connection 127.0.0.5:38817 (6 connections now open) m31003| Fri Jan 18 13:18:43.091 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:43.091 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:43.091 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:43.137 [initandlisten] connection accepted from 127.0.0.5:43148 #51 (7 connections now open) m31003| Fri Jan 18 13:18:43.246 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:43.246 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:43.246 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:43.250 [conn51] end connection 127.0.0.5:43148 (6 connections now open) m31002| Fri Jan 18 13:18:43.329 [initandlisten] connection accepted from 127.0.0.5:58344 #52 (7 connections now open) m31003| Fri Jan 18 13:18:43.420 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:43.420 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:43.420 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:43.452 [conn52] end connection 127.0.0.5:58344 (6 connections now open) m31002| Fri Jan 18 13:18:43.484 [initandlisten] connection accepted from 127.0.0.5:57535 #53 (7 connections now open) m31003| Fri Jan 18 13:18:43.534 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:43.534 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:43.534 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:43.590 [conn53] end connection 127.0.0.5:57535 (6 connections now open) m31002| Fri Jan 18 13:18:43.642 [initandlisten] connection accepted from 127.0.0.5:45683 #54 (7 connections now open) m31002| Fri Jan 18 13:18:43.746 [conn54] end connection 127.0.0.5:45683 (6 connections now open) m31003| Fri Jan 18 13:18:43.748 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:43.748 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:43.748 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:43.875 [initandlisten] connection accepted from 127.0.0.5:42447 #55 (7 connections now open) m31003| Fri Jan 18 13:18:43.975 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:43.975 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:43.975 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:44.033 [initandlisten] connection accepted from 127.0.0.5:56425 #56 (8 connections now open) m31002| Fri Jan 18 13:18:44.033 [conn55] end connection 127.0.0.5:42447 (6 connections now open) m31002| Fri Jan 18 13:18:44.200 [conn56] end connection 127.0.0.5:56425 (6 connections now open) m31003| Fri Jan 18 13:18:44.202 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:44.202 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:44.202 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:44.245 [initandlisten] connection accepted from 127.0.0.5:37232 #57 (7 connections now open) m31003| Fri Jan 18 13:18:44.341 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:44.341 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:44.341 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:44.352 [conn57] end connection 127.0.0.5:37232 (6 connections now open) m31002| Fri Jan 18 13:18:44.393 [initandlisten] connection accepted from 127.0.0.5:39511 #58 (7 connections now open) m31003| Fri Jan 18 13:18:44.584 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:44.584 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:44.584 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:44.666 [initandlisten] connection accepted from 127.0.0.5:41084 #59 (8 connections now open) m31002| Fri Jan 18 13:18:44.666 [conn58] end connection 127.0.0.5:39511 (7 connections now open) m31003| Fri Jan 18 13:18:44.717 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:44.717 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:44.717 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:44.802 [conn59] end connection 127.0.0.5:41084 (6 connections now open) m31002| Fri Jan 18 13:18:44.845 [initandlisten] connection accepted from 127.0.0.5:36422 #60 (7 connections now open) m31003| Fri Jan 18 13:18:44.936 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:44.936 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:44.936 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:44.998 [conn60] end connection 127.0.0.5:36422 (6 connections now open) m31002| Fri Jan 18 13:18:45.050 [initandlisten] connection accepted from 127.0.0.5:57121 #61 (7 connections now open) m31003| Fri Jan 18 13:18:45.183 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:45.184 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:45.184 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:45.243 [initandlisten] connection accepted from 127.0.0.5:60878 #62 (8 connections now open) m31002| Fri Jan 18 13:18:45.243 [conn61] end connection 127.0.0.5:57121 (7 connections now open) m31003| Fri Jan 18 13:18:45.322 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:45.322 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:45.322 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:45.372 [conn62] end connection 127.0.0.5:60878 (6 connections now open) m31002| Fri Jan 18 13:18:45.455 [initandlisten] connection accepted from 127.0.0.5:57460 #63 (7 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 198, "optime" : { "t" : 1358533125000, "i" : 5370 }, "optimeDate" : ISODate("2013-01-18T18:18:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 164, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 164, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 164, "optime" : { "t" : 1358533125000, "i" : 2819 }, "optimeDate" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 163, "optime" : { "t" : 1358533124000, "i" : 2377 }, "optimeDate" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 198, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:44Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358533005000, "i" : 7323 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:45Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358533124000, "i" : 2377 }, "optimeDate" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 163, "optime" : { "t" : 1358533124000, "i" : 8694 }, "optimeDate" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 197, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358533005000, "i" : 7723 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:45Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358533124000, "i" : 8694 }, "optimeDate" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:45Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 164, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 17 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 164, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 21 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 196, "optime" : { "t" : 1358533005000, "i" : 7923 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 164, "optime" : { "t" : 1358533125000, "i" : 1436 }, "optimeDate" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:45.589 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:45.589 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:45.589 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:45.589 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:18:45.589 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 163, "optime" : { "t" : 1358533124000, "i" : 2377 }, "optimeDate" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:44Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358533005000, "i" : 7323 }, "optimeDate" : ISODate("2013-01-18T18:16:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:45Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 196, "optime" : { "t" : 1358533125000, "i" : 5370 }, "optimeDate" : ISODate("2013-01-18T18:18:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31002| Fri Jan 18 13:18:45.640 [conn63] end connection 127.0.0.5:57460 (6 connections now open) m31001| Fri Jan 18 13:18:45.748 [initandlisten] connection accepted from 127.0.0.5:40161 #111 (6 connections now open) m31003| Fri Jan 18 13:18:45.807 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:45.807 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:45.807 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.3:31001 m31003| Fri Jan 18 13:18:45.807 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:45.807 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31002| Fri Jan 18 13:18:45.819 [conn44] end connection 127.0.0.5:32801 (5 connections now open) m31001| Fri Jan 18 13:18:45.901 [initandlisten] connection accepted from 127.0.0.5:37707 #112 (7 connections now open) m31001| Fri Jan 18 13:18:45.901 [initandlisten] connection accepted from 127.0.0.5:53852 #113 (8 connections now open) m31001| Fri Jan 18 13:18:45.901 [conn111] end connection 127.0.0.5:40161 (7 connections now open) m31001| Fri Jan 18 13:18:46.033 [conn112] end connection 127.0.0.5:37707 (6 connections now open) m31003| Fri Jan 18 13:18:46.036 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:46.037 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:46.037 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:46.037 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:46.137 [initandlisten] connection accepted from 127.0.0.5:51184 #114 (7 connections now open) m31003| Fri Jan 18 13:18:46.204 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:46.204 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:46.204 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:46.204 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:46.227 [conn113] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533005000|8123 } } cursorid:841187746046801 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:213831 nreturned:164650 reslen:2799070 135ms m31001| Fri Jan 18 13:18:46.233 [conn114] end connection 127.0.0.5:51184 (6 connections now open) m31001| Fri Jan 18 13:18:46.282 [initandlisten] connection accepted from 127.0.0.5:36702 #115 (7 connections now open) m31003| Fri Jan 18 13:18:46.372 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:46.372 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:46.372 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:46.372 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:46.376 [conn115] end connection 127.0.0.5:36702 (6 connections now open) m31001| Fri Jan 18 13:18:46.475 [initandlisten] connection accepted from 127.0.0.5:53160 #116 (7 connections now open) m31003| Fri Jan 18 13:18:46.609 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:46.609 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:46.609 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:46.609 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:46.649 [conn116] end connection 127.0.0.5:53160 (6 connections now open) m31001| Fri Jan 18 13:18:46.714 [initandlisten] connection accepted from 127.0.0.5:44236 #117 (7 connections now open) m31003| Fri Jan 18 13:18:46.803 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:46.803 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:46.803 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:46.803 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:46.831 [conn117] end connection 127.0.0.5:44236 (6 connections now open) m31001| Fri Jan 18 13:18:46.831 [initandlisten] connection accepted from 127.0.0.5:43729 #118 (8 connections now open) m31003| Fri Jan 18 13:18:46.991 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:46.991 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:46.991 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:46.991 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:46.993 [conn118] end connection 127.0.0.5:43729 (6 connections now open) m31001| Fri Jan 18 13:18:47.076 [initandlisten] connection accepted from 127.0.0.5:45625 #119 (7 connections now open) m31001| Fri Jan 18 13:18:47.220 [conn119] end connection 127.0.0.5:45625 (6 connections now open) m31003| Fri Jan 18 13:18:47.224 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:47.224 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:47.224 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:47.224 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:47.275 [initandlisten] connection accepted from 127.0.0.5:51349 #120 (7 connections now open) m31003| Fri Jan 18 13:18:47.415 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:47.415 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:47.415 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:47.415 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:47.476 [conn120] end connection 127.0.0.5:51349 (6 connections now open) m31001| Fri Jan 18 13:18:47.519 [initandlisten] connection accepted from 127.0.0.5:33179 #121 (7 connections now open) m31001| Fri Jan 18 13:18:47.589 [conn121] end connection 127.0.0.5:33179 (6 connections now open) m31003| Fri Jan 18 13:18:47.593 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:47.593 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:47.593 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:47.593 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:47.593 [initandlisten] connection accepted from 127.0.0.5:60828 #122 (7 connections now open) m31001| Fri Jan 18 13:18:47.694 [conn122] end connection 127.0.0.5:60828 (6 connections now open) m31003| Fri Jan 18 13:18:47.698 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:47.698 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:47.698 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:47.698 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:47.791 [initandlisten] connection accepted from 127.0.0.5:33952 #123 (7 connections now open) m31003| Fri Jan 18 13:18:47.908 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:47.908 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:47.908 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:47.908 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:47.995 [conn123] end connection 127.0.0.5:33952 (6 connections now open) m31001| Fri Jan 18 13:18:48.039 [initandlisten] connection accepted from 127.0.0.5:45557 #124 (7 connections now open) m31003| Fri Jan 18 13:18:48.205 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:48.205 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:48.205 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:48.205 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:48.263 [conn124] end connection 127.0.0.5:45557 (6 connections now open) m31001| Fri Jan 18 13:18:48.271 [initandlisten] connection accepted from 127.0.0.5:59698 #125 (7 connections now open) m31003| Fri Jan 18 13:18:48.328 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:48.328 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:48.328 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:48.328 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:48.348 [conn125] end connection 127.0.0.5:59698 (6 connections now open) m31001| Fri Jan 18 13:18:48.459 [initandlisten] connection accepted from 127.0.0.5:55767 #126 (7 connections now open) m31003| Fri Jan 18 13:18:48.562 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:48.562 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:48.562 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:48.562 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:18:48.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Fri Jan 18 13:18:48.630 [conn126] end connection 127.0.0.5:55767 (6 connections now open) m31001| Fri Jan 18 13:18:48.648 [initandlisten] connection accepted from 127.0.0.5:59088 #127 (7 connections now open) m31003| Fri Jan 18 13:18:48.720 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:48.720 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:48.720 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:48.720 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:48.810 [conn127] end connection 127.0.0.5:59088 (6 connections now open) m31001| Fri Jan 18 13:18:48.816 [initandlisten] connection accepted from 127.0.0.5:35324 #128 (7 connections now open) m31003| Fri Jan 18 13:18:48.976 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:48.976 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:48.976 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:48.976 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:49.056 [conn128] end connection 127.0.0.5:35324 (6 connections now open) m31001| Fri Jan 18 13:18:49.131 [initandlisten] connection accepted from 127.0.0.5:57547 #129 (7 connections now open) m31003| Fri Jan 18 13:18:49.278 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:49.278 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:49.278 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:49.278 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:49.366 [conn129] end connection 127.0.0.5:57547 (6 connections now open) m31001| Fri Jan 18 13:18:49.409 [initandlisten] connection accepted from 127.0.0.5:53735 #130 (7 connections now open) m31001| Fri Jan 18 13:18:49.510 [conn130] end connection 127.0.0.5:53735 (6 connections now open) m31003| Fri Jan 18 13:18:49.512 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:49.512 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:49.512 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:49.512 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:49.512 [initandlisten] connection accepted from 127.0.0.5:52343 #131 (7 connections now open) m31003| Fri Jan 18 13:18:49.562 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:49.562 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:49.562 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:49.562 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:49.609 [conn131] end connection 127.0.0.5:52343 (6 connections now open) m31001| Fri Jan 18 13:18:49.628 [initandlisten] connection accepted from 127.0.0.5:54056 #132 (7 connections now open) m31003| Fri Jan 18 13:18:49.687 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:49.687 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:49.687 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:49.687 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:49.703 [conn132] end connection 127.0.0.5:54056 (6 connections now open) m31001| Fri Jan 18 13:18:49.788 [initandlisten] connection accepted from 127.0.0.5:55908 #133 (7 connections now open) m31003| Fri Jan 18 13:18:49.849 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:49.849 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:49.849 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:49.849 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:49.865 [conn133] end connection 127.0.0.5:55908 (6 connections now open) m31001| Fri Jan 18 13:18:49.961 [initandlisten] connection accepted from 127.0.0.5:42887 #134 (7 connections now open) m31001| Fri Jan 18 13:18:50.102 [conn134] end connection 127.0.0.5:42887 (6 connections now open) m31003| Fri Jan 18 13:18:50.104 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:50.104 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:50.104 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:50.104 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:50.281 [initandlisten] connection accepted from 127.0.0.5:35976 #135 (7 connections now open) m31002| Fri Jan 18 13:18:50.309 [conn28] end connection 127.0.0.6:44178 (4 connections now open) m31002| Fri Jan 18 13:18:50.309 [initandlisten] connection accepted from 127.0.0.6:45666 #64 (5 connections now open) m31003| Fri Jan 18 13:18:50.332 [conn23] end connection 127.0.0.3:34795 (3 connections now open) m31003| Fri Jan 18 13:18:50.332 [initandlisten] connection accepted from 127.0.0.3:46663 #26 (4 connections now open) m31003| Fri Jan 18 13:18:50.384 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:50.384 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:50.384 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:50.384 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:50.384 [conn135] end connection 127.0.0.5:35976 (6 connections now open) m31001| Fri Jan 18 13:18:50.474 [initandlisten] connection accepted from 127.0.0.5:35001 #136 (7 connections now open) m31001| Fri Jan 18 13:18:50.621 [conn136] end connection 127.0.0.5:35001 (6 connections now open) m31003| Fri Jan 18 13:18:50.623 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:50.623 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:50.623 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:50.623 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:50.625 [initandlisten] connection accepted from 127.0.0.5:43286 #137 (7 connections now open) m31003| Fri Jan 18 13:18:50.704 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:50.704 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:50.704 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:50.704 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:50.748 [initandlisten] connection accepted from 127.0.0.5:51529 #138 (8 connections now open) m31001| Fri Jan 18 13:18:50.748 [conn137] end connection 127.0.0.5:43286 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 203, "optime" : { "t" : 1358533130000, "i" : 7807 }, "optimeDate" : ISODate("2013-01-18T18:18:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 169, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:50Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 169, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 169, "optime" : { "t" : 1358533129000, "i" : 2830 }, "optimeDate" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:50Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 168, "optime" : { "t" : 1358533130000, "i" : 2986 }, "optimeDate" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 203, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358533006000, "i" : 1804 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358533130000, "i" : 2979 }, "optimeDate" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 168, "optime" : { "t" : 1358533128000, "i" : 3449 }, "optimeDate" : ISODate("2013-01-18T18:18:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:50Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 202, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358533006000, "i" : 1204 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:49Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358533128000, "i" : 3435 }, "optimeDate" : ISODate("2013-01-18T18:18:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:50Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:50Z"), "myState" : 2, "syncingTo" : "127.0.0.3:31001", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 169, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:50Z"), "pingMs" : 24 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 169, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 37 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533006000, "i" : 2104 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "errmsg" : "syncing to: 127.0.0.3:31001", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 169, "optime" : { "t" : 1358533130000, "i" : 1380 }, "optimeDate" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:50.903 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:50.903 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:50.903 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:50.903 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 168, "optime" : { "t" : 1358533130000, "i" : 2962 }, "optimeDate" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:50Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358533006000, "i" : 1804 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533130000, "i" : 7807 }, "optimeDate" : ISODate("2013-01-18T18:18:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.3:31001" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:18:50.953 [initandlisten] connection accepted from 127.0.0.5:56248 #139 (8 connections now open) m31001| Fri Jan 18 13:18:50.953 [conn138] end connection 127.0.0.5:51529 (7 connections now open) m31003| Fri Jan 18 13:18:51.001 [conn24] end connection 127.0.0.4:49288 (3 connections now open) m31003| Fri Jan 18 13:18:51.001 [initandlisten] connection accepted from 127.0.0.4:52783 #27 (4 connections now open) m31003| Fri Jan 18 13:18:51.065 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:51.065 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:51.065 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:51.065 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:51.132 [conn139] end connection 127.0.0.5:56248 (6 connections now open) m31001| Fri Jan 18 13:18:51.132 [initandlisten] connection accepted from 127.0.0.5:54425 #140 (8 connections now open) m31003| Fri Jan 18 13:18:51.184 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:51.184 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:51.184 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:51.184 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:51.233 [conn140] end connection 127.0.0.5:54425 (6 connections now open) m31001| Fri Jan 18 13:18:51.248 [initandlisten] connection accepted from 127.0.0.5:60071 #141 (7 connections now open) m31003| Fri Jan 18 13:18:51.314 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:51.314 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:51.314 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:51.314 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:51.351 [conn29] end connection 127.0.0.2:52985 (6 connections now open) m31001| Fri Jan 18 13:18:51.352 [initandlisten] connection accepted from 127.0.0.2:47053 #142 (7 connections now open) m31001| Fri Jan 18 13:18:51.367 [initandlisten] connection accepted from 127.0.0.5:56900 #143 (8 connections now open) m31001| Fri Jan 18 13:18:51.367 [conn141] end connection 127.0.0.5:60071 (7 connections now open) m31002| Fri Jan 18 13:18:51.508 [conn29] end connection 127.0.0.5:58675 (4 connections now open) m31003| Fri Jan 18 13:18:51.517 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:51.517 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:51.517 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:51.517 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:51.521 [initandlisten] connection accepted from 127.0.0.5:49707 #144 (8 connections now open) m31001| Fri Jan 18 13:18:51.521 [conn143] end connection 127.0.0.5:56900 (7 connections now open) m31002| Fri Jan 18 13:18:51.623 [initandlisten] connection accepted from 127.0.0.5:35698 #65 (5 connections now open) m31003| Fri Jan 18 13:18:51.660 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:51.660 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:51.660 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:51.660 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:51.724 [conn144] end connection 127.0.0.5:49707 (6 connections now open) m31001| Fri Jan 18 13:18:51.760 [initandlisten] connection accepted from 127.0.0.5:54236 #145 (7 connections now open) m31001| Fri Jan 18 13:18:51.827 [conn145] end connection 127.0.0.5:54236 (6 connections now open) m31003| Fri Jan 18 13:18:51.831 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:51.831 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:51.831 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:51.831 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:51.831 [initandlisten] connection accepted from 127.0.0.5:58857 #146 (7 connections now open) m31003| Fri Jan 18 13:18:51.888 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:51.888 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:51.888 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:51.888 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:51.952 [conn146] end connection 127.0.0.5:58857 (6 connections now open) m31001| Fri Jan 18 13:18:51.952 [initandlisten] connection accepted from 127.0.0.5:51727 #147 (8 connections now open) m31001| Fri Jan 18 13:18:52.164 [conn147] end connection 127.0.0.5:51727 (6 connections now open) m31003| Fri Jan 18 13:18:52.167 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:52.167 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:52.167 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:52.167 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:52.296 [initandlisten] connection accepted from 127.0.0.5:60957 #148 (7 connections now open) m31000| Fri Jan 18 13:18:52.309 [conn43] end connection 127.0.0.6:49165 (11 connections now open) m31000| Fri Jan 18 13:18:52.309 [initandlisten] connection accepted from 127.0.0.6:43398 #48 (12 connections now open) m31000| Fri Jan 18 13:18:52.333 [conn44] end connection 127.0.0.3:46112 (11 connections now open) m31000| Fri Jan 18 13:18:52.333 [initandlisten] connection accepted from 127.0.0.3:52472 #49 (12 connections now open) m31003| Fri Jan 18 13:18:52.405 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:52.406 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:52.406 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:52.406 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:52.495 [conn148] end connection 127.0.0.5:60957 (6 connections now open) m31001| Fri Jan 18 13:18:52.586 [initandlisten] connection accepted from 127.0.0.5:53040 #149 (7 connections now open) m31003| Fri Jan 18 13:18:52.638 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:52.638 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:52.638 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:52.638 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:52.660 [conn149] end connection 127.0.0.5:53040 (6 connections now open) m31001| Fri Jan 18 13:18:52.730 [initandlisten] connection accepted from 127.0.0.5:40753 #150 (7 connections now open) m31003| Fri Jan 18 13:18:52.826 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:52.826 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:52.826 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:52.826 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:52.839 [conn150] end connection 127.0.0.5:40753 (6 connections now open) m31001| Fri Jan 18 13:18:52.993 [initandlisten] connection accepted from 127.0.0.5:54924 #151 (7 connections now open) m31000| Fri Jan 18 13:18:53.001 [conn45] end connection 127.0.0.4:36116 (11 connections now open) m31000| Fri Jan 18 13:18:53.001 [initandlisten] connection accepted from 127.0.0.4:42807 #50 (13 connections now open) m31003| Fri Jan 18 13:18:53.144 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:53.144 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:53.144 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:53.144 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:53.165 [conn151] end connection 127.0.0.5:54924 (6 connections now open) m31001| Fri Jan 18 13:18:53.177 [initandlisten] connection accepted from 127.0.0.5:50439 #152 (7 connections now open) m31001| Fri Jan 18 13:18:53.327 [conn152] end connection 127.0.0.5:50439 (6 connections now open) m31003| Fri Jan 18 13:18:53.331 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:53.331 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:53.331 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:53.331 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:53.336 [initandlisten] connection accepted from 127.0.0.5:49375 #153 (7 connections now open) m31003| Fri Jan 18 13:18:53.401 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:53.401 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:53.401 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:53.401 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:53.442 [conn153] end connection 127.0.0.5:49375 (6 connections now open) m31001| Fri Jan 18 13:18:53.515 [initandlisten] connection accepted from 127.0.0.5:36229 #154 (7 connections now open) m31001| Fri Jan 18 13:18:53.652 [conn154] end connection 127.0.0.5:36229 (6 connections now open) m31003| Fri Jan 18 13:18:53.656 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:53.656 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:53.656 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:18:53.656 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31002| Fri Jan 18 13:18:53.729 [initandlisten] connection accepted from 127.0.0.5:55413 #66 (6 connections now open) m31003| Fri Jan 18 13:18:53.784 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 m31003| Fri Jan 18 13:18:53.784 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:53.785 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:53.785 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31001| Fri Jan 18 13:18:53.801 [conn113] end connection 127.0.0.5:53852 (5 connections now open) m31002| Fri Jan 18 13:18:53.840 [conn66] end connection 127.0.0.5:55413 (5 connections now open) m31002| Fri Jan 18 13:18:53.933 [initandlisten] connection accepted from 127.0.0.5:56905 #67 (6 connections now open) m31002| Fri Jan 18 13:18:53.933 [initandlisten] connection accepted from 127.0.0.5:56489 #68 (7 connections now open) m31003| Fri Jan 18 13:18:54.035 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:54.035 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:54.035 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:54.114 [initandlisten] connection accepted from 127.0.0.5:50474 #69 (8 connections now open) m31002| Fri Jan 18 13:18:54.114 [conn68] end connection 127.0.0.5:56489 (7 connections now open) m31003| Fri Jan 18 13:18:54.227 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:54.227 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:54.227 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:54.254 [conn67] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533006000|3804 } } cursorid:872352342150213 ntoreturn:0 keyUpdates:0 numYields: 2 locks(micros) r:243073 nreturned:160450 reslen:2727670 144ms m31002| Fri Jan 18 13:18:54.285 [conn69] end connection 127.0.0.5:50474 (6 connections now open) m31002| Fri Jan 18 13:18:54.285 [initandlisten] connection accepted from 127.0.0.5:41317 #70 (8 connections now open) m31003| Fri Jan 18 13:18:54.372 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:54.372 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:54.372 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:54.374 [conn70] end connection 127.0.0.5:41317 (6 connections now open) m31002| Fri Jan 18 13:18:54.384 [initandlisten] connection accepted from 127.0.0.5:45503 #71 (7 connections now open) m31003| Fri Jan 18 13:18:54.510 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:54.510 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:54.510 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:54.530 [initandlisten] connection accepted from 127.0.0.5:59463 #72 (8 connections now open) m31002| Fri Jan 18 13:18:54.601 [conn71] end connection 127.0.0.5:45503 (7 connections now open) m31003| Fri Jan 18 13:18:54.675 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:54.675 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:54.675 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:54.756 [conn72] end connection 127.0.0.5:59463 (6 connections now open) m31002| Fri Jan 18 13:18:54.756 [initandlisten] connection accepted from 127.0.0.5:44455 #73 (8 connections now open) m31003| Fri Jan 18 13:18:54.933 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:54.933 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:54.933 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:54.951 [initandlisten] connection accepted from 127.0.0.5:37489 #74 (8 connections now open) m31002| Fri Jan 18 13:18:54.951 [conn73] end connection 127.0.0.5:44455 (6 connections now open) m31002| Fri Jan 18 13:18:55.024 [conn74] end connection 127.0.0.5:37489 (6 connections now open) m31003| Fri Jan 18 13:18:55.027 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:55.027 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:55.027 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:55.107 [initandlisten] connection accepted from 127.0.0.5:48359 #75 (7 connections now open) m31003| Fri Jan 18 13:18:55.248 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:55.248 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:55.248 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:55.261 [conn75] end connection 127.0.0.5:48359 (6 connections now open) m31002| Fri Jan 18 13:18:55.281 [initandlisten] connection accepted from 127.0.0.5:37729 #76 (7 connections now open) m31003| Fri Jan 18 13:18:55.453 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:55.453 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:55.453 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:55.537 [initandlisten] connection accepted from 127.0.0.5:35485 #77 (8 connections now open) m31002| Fri Jan 18 13:18:55.537 [conn76] end connection 127.0.0.5:37729 (6 connections now open) m31003| Fri Jan 18 13:18:55.769 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:55.769 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:55.769 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:55.828 [conn77] end connection 127.0.0.5:35485 (6 connections now open) m31002| Fri Jan 18 13:18:55.859 [initandlisten] connection accepted from 127.0.0.5:37748 #78 (7 connections now open) m31003| Fri Jan 18 13:18:55.940 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:55.940 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:55.940 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:56.015 [initandlisten] connection accepted from 127.0.0.5:39384 #79 (8 connections now open) m31002| Fri Jan 18 13:18:56.015 [conn78] end connection 127.0.0.5:37748 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 209, "optime" : { "t" : 1358533136000, "i" : 1136 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 175, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 175, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 175, "optime" : { "t" : 1358533135000, "i" : 3076 }, "optimeDate" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 174, "optime" : { "t" : 1358533134000, "i" : 3265 }, "optimeDate" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 209, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 174, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:54Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 174, "optime" : { "t" : 1358533006000, "i" : 4004 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:55Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 174, "optime" : { "t" : 1358533134000, "i" : 3245 }, "optimeDate" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 174, "optime" : { "t" : 1358533134000, "i" : 9429 }, "optimeDate" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 174, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 208, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 174, "optime" : { "t" : 1358533006000, "i" : 4404 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:55Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 174, "optime" : { "t" : 1358533134000, "i" : 9409 }, "optimeDate" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 m31002| Fri Jan 18 13:18:56.147 [conn79] end connection 127.0.0.5:39384 (6 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:56Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 175, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 45 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 175, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 44 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533006000, "i" : 4904 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 175, "optime" : { "t" : 1358533135000, "i" : 255 }, "optimeDate" : ISODate("2013-01-18T18:18:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:56.149 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:56.149 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:56.149 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:18:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 174, "optime" : { "t" : 1358533134000, "i" : 2969 }, "optimeDate" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 174, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 174, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:54Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 174, "optime" : { "t" : 1358533006000, "i" : 4004 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:18:55Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533136000, "i" : 1136 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31002| Fri Jan 18 13:18:56.196 [initandlisten] connection accepted from 127.0.0.5:37555 #80 (7 connections now open) m31003| Fri Jan 18 13:18:56.299 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:56.299 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:56.299 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:56.317 [conn80] end connection 127.0.0.5:37555 (6 connections now open) m31002| Fri Jan 18 13:18:56.317 [initandlisten] connection accepted from 127.0.0.5:44658 #81 (8 connections now open) m31000| Fri Jan 18 13:18:56.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:18:56.487 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:56.487 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:56.487 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:56.504 [conn81] end connection 127.0.0.5:44658 (6 connections now open) m31002| Fri Jan 18 13:18:56.604 [initandlisten] connection accepted from 127.0.0.5:59451 #82 (7 connections now open) m31002| Fri Jan 18 13:18:56.721 [conn82] end connection 127.0.0.5:59451 (6 connections now open) m31003| Fri Jan 18 13:18:56.725 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:56.725 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:56.725 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:56.799 [initandlisten] connection accepted from 127.0.0.5:36787 #83 (7 connections now open) m31002| Fri Jan 18 13:18:56.938 [conn83] end connection 127.0.0.5:36787 (6 connections now open) m31003| Fri Jan 18 13:18:56.943 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:56.943 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:56.943 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:57.038 [initandlisten] connection accepted from 127.0.0.5:58599 #84 (7 connections now open) m31003| Fri Jan 18 13:18:57.189 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:57.189 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:57.189 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:57.203 [conn84] end connection 127.0.0.5:58599 (6 connections now open) m31002| Fri Jan 18 13:18:57.295 [initandlisten] connection accepted from 127.0.0.5:33543 #85 (7 connections now open) m31003| Fri Jan 18 13:18:57.371 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:57.371 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:57.371 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:57.380 [conn85] end connection 127.0.0.5:33543 (6 connections now open) m31002| Fri Jan 18 13:18:57.380 [initandlisten] connection accepted from 127.0.0.5:54500 #86 (8 connections now open) m31003| Fri Jan 18 13:18:57.502 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:57.502 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:57.502 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:18:57.536 [initandlisten] connection accepted from 127.0.0.5:54378 #87 (8 connections now open) m31002| Fri Jan 18 13:18:57.567 [conn86] end connection 127.0.0.5:54500 (7 connections now open) m31003| Fri Jan 18 13:18:57.685 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:57.685 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:57.685 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:57.685 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:18:57.685 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31001| Fri Jan 18 13:18:57.708 [initandlisten] connection accepted from 127.0.0.5:50811 #155 (6 connections now open) m31002| Fri Jan 18 13:18:57.748 [conn87] end connection 127.0.0.5:54378 (6 connections now open) m31003| Fri Jan 18 13:18:57.779 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:57.779 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:57.779 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.3:31001 m31003| Fri Jan 18 13:18:57.779 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:57.779 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31002| Fri Jan 18 13:18:57.792 [conn67] end connection 127.0.0.5:56905 (5 connections now open) m31001| Fri Jan 18 13:18:57.838 [conn155] end connection 127.0.0.5:50811 (5 connections now open) m31001| Fri Jan 18 13:18:57.838 [initandlisten] connection accepted from 127.0.0.5:51255 #156 (7 connections now open) m31001| Fri Jan 18 13:18:57.838 [initandlisten] connection accepted from 127.0.0.5:52153 #157 (8 connections now open) m31003| Fri Jan 18 13:18:58.038 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:58.038 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:58.038 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:58.038 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:58.043 [conn157] end connection 127.0.0.5:52153 (6 connections now open) m31001| Fri Jan 18 13:18:58.054 [initandlisten] connection accepted from 127.0.0.5:43448 #158 (7 connections now open) m31003| Fri Jan 18 13:18:58.156 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:58.156 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:58.156 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:58.156 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:58.186 [conn156] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533006000|5904 } } cursorid:891954861495520 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:229476 nreturned:158350 reslen:2691970 131ms m31001| Fri Jan 18 13:18:58.216 [conn158] end connection 127.0.0.5:43448 (6 connections now open) m31001| Fri Jan 18 13:18:58.265 [initandlisten] connection accepted from 127.0.0.5:59013 #159 (7 connections now open) m31003| Fri Jan 18 13:18:58.369 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:58.369 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:58.369 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:58.369 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:58.372 [initandlisten] connection accepted from 127.0.0.5:49265 #160 (8 connections now open) m31001| Fri Jan 18 13:18:58.439 [conn159] end connection 127.0.0.5:59013 (7 connections now open) m31003| Fri Jan 18 13:18:58.525 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:58.525 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:58.525 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:58.525 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:58.549 [conn160] end connection 127.0.0.5:49265 (6 connections now open) m31001| Fri Jan 18 13:18:58.625 [initandlisten] connection accepted from 127.0.0.5:46768 #161 (7 connections now open) m31003| Fri Jan 18 13:18:58.682 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:58.682 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:58.682 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:58.682 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:58.759 [conn161] end connection 127.0.0.5:46768 (6 connections now open) m31001| Fri Jan 18 13:18:58.817 [initandlisten] connection accepted from 127.0.0.5:53567 #162 (7 connections now open) m31003| Fri Jan 18 13:18:58.888 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:58.888 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:58.888 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:58.888 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:58.962 [conn162] end connection 127.0.0.5:53567 (6 connections now open) m31001| Fri Jan 18 13:18:58.982 [initandlisten] connection accepted from 127.0.0.5:53801 #163 (7 connections now open) m31003| Fri Jan 18 13:18:59.091 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:59.091 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:59.091 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:59.091 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:59.163 [conn163] end connection 127.0.0.5:53801 (6 connections now open) m31001| Fri Jan 18 13:18:59.163 [initandlisten] connection accepted from 127.0.0.5:51923 #164 (7 connections now open) m31003| Fri Jan 18 13:18:59.263 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:59.263 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:59.263 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:59.263 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:59.281 [conn164] end connection 127.0.0.5:51923 (6 connections now open) m31001| Fri Jan 18 13:18:59.369 [initandlisten] connection accepted from 127.0.0.5:40008 #165 (7 connections now open) m31003| Fri Jan 18 13:18:59.506 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:59.506 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:59.506 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:59.506 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:59.568 [initandlisten] connection accepted from 127.0.0.5:50659 #166 (8 connections now open) m31001| Fri Jan 18 13:18:59.568 [conn165] end connection 127.0.0.5:40008 (6 connections now open) m31003| Fri Jan 18 13:18:59.729 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:59.729 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:59.729 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:59.729 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:59.785 [conn166] end connection 127.0.0.5:50659 (6 connections now open) m31001| Fri Jan 18 13:18:59.789 [initandlisten] connection accepted from 127.0.0.5:43749 #167 (7 connections now open) m31003| Fri Jan 18 13:18:59.903 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:18:59.903 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:18:59.903 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:18:59.903 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:18:59.990 [initandlisten] connection accepted from 127.0.0.5:38570 #168 (8 connections now open) m31001| Fri Jan 18 13:18:59.990 [conn167] end connection 127.0.0.5:43749 (7 connections now open) m31003| Fri Jan 18 13:19:00.080 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:00.080 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:00.080 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:00.080 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:00.128 [conn168] end connection 127.0.0.5:38570 (6 connections now open) m31001| Fri Jan 18 13:19:00.192 [initandlisten] connection accepted from 127.0.0.5:45629 #169 (7 connections now open) m31001| Fri Jan 18 13:19:00.248 [conn169] end connection 127.0.0.5:45629 (6 connections now open) m31003| Fri Jan 18 13:19:00.250 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:00.250 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:00.250 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:00.250 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:00.338 [initandlisten] connection accepted from 127.0.0.5:40055 #170 (7 connections now open) m31003| Fri Jan 18 13:19:00.431 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:00.431 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:00.431 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:00.431 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:00.515 [conn170] end connection 127.0.0.5:40055 (6 connections now open) m31001| Fri Jan 18 13:19:00.515 [initandlisten] connection accepted from 127.0.0.5:45932 #171 (8 connections now open) m31003| Fri Jan 18 13:19:00.622 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:00.622 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:00.622 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:00.622 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:00.696 [conn171] end connection 127.0.0.5:45932 (6 connections now open) m31001| Fri Jan 18 13:19:00.735 [initandlisten] connection accepted from 127.0.0.5:33553 #172 (7 connections now open) m31001| Fri Jan 18 13:19:00.839 [conn172] end connection 127.0.0.5:33553 (6 connections now open) m31003| Fri Jan 18 13:19:00.841 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:00.841 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:00.841 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:00.841 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:00.908 [initandlisten] connection accepted from 127.0.0.5:34762 #173 (7 connections now open) m31003| Fri Jan 18 13:19:01.008 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:01.008 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:01.008 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:01.008 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:01.043 [conn173] end connection 127.0.0.5:34762 (6 connections now open) m31001| Fri Jan 18 13:19:01.060 [initandlisten] connection accepted from 127.0.0.5:45044 #174 (7 connections now open) m31001| Fri Jan 18 13:19:01.159 [conn174] end connection 127.0.0.5:45044 (6 connections now open) m31003| Fri Jan 18 13:19:01.164 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:01.164 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:01.164 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:01.164 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:01.165 [initandlisten] connection accepted from 127.0.0.5:47582 #175 (7 connections now open) m31003| Fri Jan 18 13:19:01.325 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:01.325 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:01.325 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:01.325 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 STATUS FROM : 127.0.0.2:31000 m31001| Fri Jan 18 13:19:01.391 [conn175] end connection 127.0.0.5:47582 (6 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 214, "optime" : { "t" : 1358533141000, "i" : 3152 }, "optimeDate" : ISODate("2013-01-18T18:19:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358533141000, "i" : 2254 }, "optimeDate" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 m31001| Fri Jan 18 13:19:01.394 [initandlisten] connection accepted from 127.0.0.5:49383 #176 (7 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 179, "optime" : { "t" : 1358533140000, "i" : 3424 }, "optimeDate" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 214, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 179, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 179, "optime" : { "t" : 1358533006000, "i" : 7204 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 179, "optime" : { "t" : 1358533140000, "i" : 3387 }, "optimeDate" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 179, "optime" : { "t" : 1358533140000, "i" : 9873 }, "optimeDate" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 179, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 213, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 179, "optime" : { "t" : 1358533006000, "i" : 7504 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 179, "optime" : { "t" : 1358533140000, "i" : 9850 }, "optimeDate" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 m31003| Fri Jan 18 13:19:01.535 [conn1] command admin.$cmd command: { replSetGetStatus: true } ntoreturn:1 keyUpdates:0 reslen:1149 139ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:01Z"), "myState" : 2, "syncingTo" : "127.0.0.3:31001", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:00Z"), "pingMs" : 29 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:00Z"), "pingMs" : 34 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533006000, "i" : 7804 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "errmsg" : "syncing to: 127.0.0.3:31001", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358533140000, "i" : 922 }, "optimeDate" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:01.537 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:01.537 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:01.537 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:01.537 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 179, "optime" : { "t" : 1358533140000, "i" : 2908 }, "optimeDate" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 179, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 179, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 179, "optime" : { "t" : 1358533006000, "i" : 7204 }, "optimeDate" : ISODate("2013-01-18T18:16:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533141000, "i" : 3152 }, "optimeDate" : ISODate("2013-01-18T18:19:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.3:31001" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:19:01.561 [initandlisten] connection accepted from 127.0.0.5:43926 #177 (8 connections now open) m31001| Fri Jan 18 13:19:01.561 [conn176] end connection 127.0.0.5:49383 (7 connections now open) m31001| Fri Jan 18 13:19:01.660 [conn177] end connection 127.0.0.5:43926 (6 connections now open) m31003| Fri Jan 18 13:19:01.664 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:01.664 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:01.664 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:01.664 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:01.743 [initandlisten] connection accepted from 127.0.0.5:59674 #178 (7 connections now open) m31003| Fri Jan 18 13:19:01.830 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:01.830 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:01.830 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:01.830 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:01.916 [conn178] end connection 127.0.0.5:59674 (6 connections now open) m31001| Fri Jan 18 13:19:01.916 [initandlisten] connection accepted from 127.0.0.5:57638 #179 (8 connections now open) m31003| Fri Jan 18 13:19:01.969 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:01.969 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:01.969 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:01.969 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:02.064 [initandlisten] connection accepted from 127.0.0.5:58184 #180 (8 connections now open) m31001| Fri Jan 18 13:19:02.064 [conn179] end connection 127.0.0.5:57638 (7 connections now open) m31003| Fri Jan 18 13:19:02.126 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:02.126 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:02.126 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:02.126 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:02.178 [conn180] end connection 127.0.0.5:58184 (6 connections now open) m31001| Fri Jan 18 13:19:02.223 [initandlisten] connection accepted from 127.0.0.5:57747 #181 (7 connections now open) m31003| Fri Jan 18 13:19:02.364 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:02.364 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:02.364 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:02.364 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:02.414 [conn181] end connection 127.0.0.5:57747 (6 connections now open) m31001| Fri Jan 18 13:19:02.414 [initandlisten] connection accepted from 127.0.0.5:41929 #182 (8 connections now open) m31003| Fri Jan 18 13:19:02.608 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:02.608 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:02.608 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:02.608 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31004| Fri Jan 18 13:19:02.618 [conn25] end connection 127.0.0.5:35143 (4 connections now open) m31001| Fri Jan 18 13:19:02.659 [conn182] end connection 127.0.0.5:41929 (6 connections now open) m31001| Fri Jan 18 13:19:02.721 [initandlisten] connection accepted from 127.0.0.5:37857 #183 (7 connections now open) m31003| Fri Jan 18 13:19:02.864 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:02.864 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:02.864 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:02.864 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:02.961 [initandlisten] connection accepted from 127.0.0.5:47941 #184 (8 connections now open) m31001| Fri Jan 18 13:19:02.961 [conn183] end connection 127.0.0.5:37857 (7 connections now open) m31003| Fri Jan 18 13:19:03.026 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:03.026 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:03.026 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:03.026 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:03.103 [initandlisten] connection accepted from 127.0.0.5:45169 #185 (8 connections now open) m31001| Fri Jan 18 13:19:03.103 [conn184] end connection 127.0.0.5:47941 (6 connections now open) m31004| Fri Jan 18 13:19:03.118 [initandlisten] connection accepted from 127.0.0.5:51869 #29 (5 connections now open) m31003| Fri Jan 18 13:19:03.171 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:03.171 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:03.171 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:03.171 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:03.254 [initandlisten] connection accepted from 127.0.0.5:37574 #186 (8 connections now open) m31001| Fri Jan 18 13:19:03.254 [conn185] end connection 127.0.0.5:45169 (7 connections now open) m31004| Fri Jan 18 13:19:03.353 [conn26] end connection 127.0.0.2:56902 (4 connections now open) m31004| Fri Jan 18 13:19:03.353 [initandlisten] connection accepted from 127.0.0.2:58978 #30 (5 connections now open) m31001| Fri Jan 18 13:19:03.380 [conn186] end connection 127.0.0.5:37574 (6 connections now open) m31003| Fri Jan 18 13:19:03.382 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:03.382 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:03.382 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:03.382 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:03.425 [initandlisten] connection accepted from 127.0.0.5:54304 #187 (7 connections now open) m31003| Fri Jan 18 13:19:03.480 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:03.480 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:03.480 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:03.480 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:03.571 [initandlisten] connection accepted from 127.0.0.5:39199 #188 (8 connections now open) m31001| Fri Jan 18 13:19:03.571 [conn187] end connection 127.0.0.5:54304 (6 connections now open) m31003| Fri Jan 18 13:19:03.698 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:03.698 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:03.698 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:03.698 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:03.704 [conn188] end connection 127.0.0.5:39199 (6 connections now open) m31001| Fri Jan 18 13:19:03.781 [initandlisten] connection accepted from 127.0.0.5:40291 #189 (7 connections now open) m31001| Fri Jan 18 13:19:03.886 [conn189] end connection 127.0.0.5:40291 (6 connections now open) m31003| Fri Jan 18 13:19:03.889 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:03.889 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:03.889 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:19:03.889 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31002| Fri Jan 18 13:19:03.946 [initandlisten] connection accepted from 127.0.0.5:54439 #88 (6 connections now open) m31003| Fri Jan 18 13:19:04.059 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 m31001| Fri Jan 18 13:19:04.100 [conn156] end connection 127.0.0.5:51255 (5 connections now open) m31002| Fri Jan 18 13:19:04.113 [conn88] end connection 127.0.0.5:54439 (5 connections now open) m31002| Fri Jan 18 13:19:04.113 [initandlisten] connection accepted from 127.0.0.5:56830 #89 (7 connections now open) m31003| Fri Jan 18 13:19:04.183 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:04.183 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:04.183 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:04.268 [initandlisten] connection accepted from 127.0.0.5:37693 #90 (7 connections now open) m31003| Fri Jan 18 13:19:04.311 [conn25] end connection 127.0.0.6:58508 (3 connections now open) m31003| Fri Jan 18 13:19:04.311 [initandlisten] connection accepted from 127.0.0.6:60806 #28 (4 connections now open) m31003| Fri Jan 18 13:19:04.324 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:04.324 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:04.324 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31004| Fri Jan 18 13:19:04.334 [conn27] end connection 127.0.0.3:49876 (4 connections now open) m31004| Fri Jan 18 13:19:04.334 [initandlisten] connection accepted from 127.0.0.3:50352 #31 (5 connections now open) m31002| Fri Jan 18 13:19:04.419 [conn90] end connection 127.0.0.5:37693 (6 connections now open) m31002| Fri Jan 18 13:19:04.419 [initandlisten] connection accepted from 127.0.0.5:55439 #91 (8 connections now open) m31002| Fri Jan 18 13:19:04.561 [conn89] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533010000|244 } } cursorid:916367163188489 ntoreturn:0 keyUpdates:0 numYields: 2 locks(micros) r:279231 nreturned:154950 reslen:2634170 142ms m31003| Fri Jan 18 13:19:04.570 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:04.570 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:04.570 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:04.612 [initandlisten] connection accepted from 127.0.0.5:56784 #92 (8 connections now open) m31002| Fri Jan 18 13:19:04.612 [conn91] end connection 127.0.0.5:55439 (7 connections now open) m31003| Fri Jan 18 13:19:04.766 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:04.766 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:04.766 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:04.783 [initandlisten] connection accepted from 127.0.0.5:51370 #93 (8 connections now open) m31002| Fri Jan 18 13:19:04.809 [conn92] end connection 127.0.0.5:56784 (7 connections now open) m31003| Fri Jan 18 13:19:04.965 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:04.965 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:04.965 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31004| Fri Jan 18 13:19:05.003 [conn28] end connection 127.0.0.4:46753 (4 connections now open) m31004| Fri Jan 18 13:19:05.003 [initandlisten] connection accepted from 127.0.0.4:45719 #32 (5 connections now open) m31002| Fri Jan 18 13:19:05.062 [initandlisten] connection accepted from 127.0.0.5:60459 #94 (8 connections now open) m31002| Fri Jan 18 13:19:05.062 [conn93] end connection 127.0.0.5:51370 (7 connections now open) m31003| Fri Jan 18 13:19:05.192 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:05.192 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:05.192 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:05.281 [initandlisten] connection accepted from 127.0.0.5:35900 #95 (8 connections now open) m31002| Fri Jan 18 13:19:05.281 [conn94] end connection 127.0.0.5:60459 (6 connections now open) m31002| Fri Jan 18 13:19:05.354 [conn40] end connection 127.0.0.2:52133 (6 connections now open) m31002| Fri Jan 18 13:19:05.354 [initandlisten] connection accepted from 127.0.0.2:47652 #96 (7 connections now open) m31003| Fri Jan 18 13:19:05.401 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:05.401 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:05.401 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:05.436 [conn95] end connection 127.0.0.5:35900 (6 connections now open) m31002| Fri Jan 18 13:19:05.447 [initandlisten] connection accepted from 127.0.0.5:43076 #97 (7 connections now open) m31002| Fri Jan 18 13:19:05.636 [conn97] end connection 127.0.0.5:43076 (6 connections now open) m31003| Fri Jan 18 13:19:05.637 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:05.637 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:05.637 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:05.787 [initandlisten] connection accepted from 127.0.0.5:52016 #98 (7 connections now open) m31003| Fri Jan 18 13:19:05.938 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:05.938 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:05.938 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:05.938 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:19:05.938 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31001| Fri Jan 18 13:19:05.938 [initandlisten] connection accepted from 127.0.0.5:45225 #190 (6 connections now open) m31001| Fri Jan 18 13:19:05.984 [conn190] end connection 127.0.0.5:45225 (5 connections now open) m31003| Fri Jan 18 13:19:05.986 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.3:31001 m31003| Fri Jan 18 13:19:05.986 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:05.986 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:05.986 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:05.986 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31002| Fri Jan 18 13:19:06.013 [conn98] end connection 127.0.0.5:52016 (6 connections now open) m31001| Fri Jan 18 13:19:06.068 [initandlisten] connection accepted from 127.0.0.5:43364 #191 (6 connections now open) m31002| Fri Jan 18 13:19:06.073 [conn89] end connection 127.0.0.5:56830 (5 connections now open) m31001| Fri Jan 18 13:19:06.126 [initandlisten] connection accepted from 127.0.0.5:52376 #192 (7 connections now open) m31001| Fri Jan 18 13:19:06.245 [conn191] end connection 127.0.0.5:43364 (6 connections now open) m31001| Fri Jan 18 13:19:06.311 [conn82] end connection 127.0.0.6:52681 (5 connections now open) m31001| Fri Jan 18 13:19:06.311 [initandlisten] connection accepted from 127.0.0.6:39469 #193 (6 connections now open) m31000| Fri Jan 18 13:19:06.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31002| Fri Jan 18 13:19:06.335 [conn41] end connection 127.0.0.3:55347 (4 connections now open) m31002| Fri Jan 18 13:19:06.335 [initandlisten] connection accepted from 127.0.0.3:40898 #99 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:19:06.840 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:1460467 730ms m31003| Fri Jan 18 13:19:06.842 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:06.842 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:06.842 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:06.842 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:06.912 [initandlisten] connection accepted from 127.0.0.5:47498 #194 (7 connections now open) m31001| Fri Jan 18 13:19:07.003 [conn87] end connection 127.0.0.4:58333 (6 connections now open) m31001| Fri Jan 18 13:19:07.003 [initandlisten] connection accepted from 127.0.0.4:45186 #195 (7 connections now open) m31001| Fri Jan 18 13:19:07.044 [conn192] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533010000|1144 } } cursorid:927052413651298 ntoreturn:0 keyUpdates:0 locks(micros) r:131580 nreturned:154050 reslen:2618870 131ms m31003| Fri Jan 18 13:19:07.098 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:07.098 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:07.098 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:07.098 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:07.112 [initandlisten] connection accepted from 127.0.0.5:57965 #196 (8 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 220, "optime" : { "t" : 1358533147000, "i" : 3132 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 186, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 186, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:07Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 186, "optime" : { "t" : 1358533145000, "i" : 2541 }, "optimeDate" : ISODate("2013-01-18T18:19:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 185, "optime" : { "t" : 1358533146000, "i" : 1347 }, "optimeDate" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 220, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:07Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358533010000, "i" : 1144 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358533146000, "i" : 1341 }, "optimeDate" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 185, "optime" : { "t" : 1358533146000, "i" : 4554 }, "optimeDate" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 219, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358533010000, "i" : 1244 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358533146000, "i" : 4493 }, "optimeDate" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 m31001| Fri Jan 18 13:19:07.163 [conn194] end connection 127.0.0.5:47498 (7 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:07Z"), "myState" : 2, "syncingTo" : "127.0.0.3:31001", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 186, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:06Z"), "pingMs" : 23 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 186, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:07Z"), "pingMs" : 39 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358533010000, "i" : 1344 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "errmsg" : "syncing to: 127.0.0.3:31001", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 186, "optime" : { "t" : 1358533145000, "i" : 5284 }, "optimeDate" : ISODate("2013-01-18T18:19:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:07.169 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:07.169 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:07.169 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:07.169 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 185, "optime" : { "t" : 1358533146000, "i" : 1347 }, "optimeDate" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358533077000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:17:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:07Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358533010000, "i" : 1144 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358533147000, "i" : 3132 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.3:31001" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:19:07.221 [conn196] end connection 127.0.0.5:57965 (6 connections now open) m31001| Fri Jan 18 13:19:07.240 [initandlisten] connection accepted from 127.0.0.5:36108 #197 (7 connections now open) m31003| Fri Jan 18 13:19:07.304 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:07.304 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:07.304 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:07.304 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:07.396 [initandlisten] connection accepted from 127.0.0.5:42040 #198 (8 connections now open) m31001| Fri Jan 18 13:19:07.396 [conn197] end connection 127.0.0.5:36108 (6 connections now open) m31003| Fri Jan 18 13:19:07.508 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:07.508 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:07.508 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:07.508 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31000| Fri Jan 18 13:19:07.525 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:136779 nreturned:160000 reslen:2720020 136ms m31001| Fri Jan 18 13:19:07.557 [conn198] end connection 127.0.0.5:42040 (6 connections now open) m31001| Fri Jan 18 13:19:07.604 [initandlisten] connection accepted from 127.0.0.5:54867 #199 (7 connections now open) m31001| Fri Jan 18 13:19:07.697 [conn89] end connection 127.0.0.5:55319 (6 connections now open) m31001| Fri Jan 18 13:19:07.794 [initandlisten] connection accepted from 127.0.0.5:58358 #200 (7 connections now open) m31000| Fri Jan 18 13:19:07.859 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 numYields: 178 locks(micros) r:266056 nreturned:160000 reslen:2720020 133ms m31003| Fri Jan 18 13:19:07.913 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:07.913 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:07.913 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:07.913 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:08.000 [initandlisten] connection accepted from 127.0.0.5:52169 #201 (8 connections now open) m31001| Fri Jan 18 13:19:08.000 [conn199] end connection 127.0.0.5:54867 (7 connections now open) m31003| Fri Jan 18 13:19:08.193 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:08.193 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:08.193 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:08.193 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:08.239 [conn201] end connection 127.0.0.5:52169 (6 connections now open) m31001| Fri Jan 18 13:19:08.274 [initandlisten] connection accepted from 127.0.0.5:41861 #202 (7 connections now open) m31003| Fri Jan 18 13:19:08.391 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:08.391 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:08.391 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:08.391 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:08.469 [initandlisten] connection accepted from 127.0.0.5:39884 #203 (8 connections now open) m31001| Fri Jan 18 13:19:08.469 [conn202] end connection 127.0.0.5:41861 (6 connections now open) m31001| Fri Jan 18 13:19:08.594 [conn203] end connection 127.0.0.5:39884 (6 connections now open) m31003| Fri Jan 18 13:19:08.596 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:08.596 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:08.596 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:08.596 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:08.609 [initandlisten] connection accepted from 127.0.0.5:41871 #204 (7 connections now open) m31003| Fri Jan 18 13:19:08.736 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:08.736 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:08.737 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:08.737 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:08.810 [conn204] end connection 127.0.0.5:41871 (6 connections now open) m31001| Fri Jan 18 13:19:08.810 [initandlisten] connection accepted from 127.0.0.5:34422 #205 (8 connections now open) m31003| Fri Jan 18 13:19:08.859 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:08.859 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:08.859 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:08.859 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:08.909 [conn205] end connection 127.0.0.5:34422 (6 connections now open) m31001| Fri Jan 18 13:19:08.990 [initandlisten] connection accepted from 127.0.0.5:56066 #206 (7 connections now open) m31001| Fri Jan 18 13:19:09.182 [conn206] end connection 127.0.0.5:56066 (6 connections now open) m31003| Fri Jan 18 13:19:09.184 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:09.184 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:09.184 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:09.184 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:09.219 [initandlisten] connection accepted from 127.0.0.5:33120 #207 (7 connections now open) m31003| Fri Jan 18 13:19:09.365 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:09.365 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:09.365 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:09.365 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:09.435 [conn207] end connection 127.0.0.5:33120 (6 connections now open) m31001| Fri Jan 18 13:19:09.435 [initandlisten] connection accepted from 127.0.0.5:59535 #208 (8 connections now open) m31001| Fri Jan 18 13:19:09.552 [conn208] end connection 127.0.0.5:59535 (6 connections now open) m31003| Fri Jan 18 13:19:09.556 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:09.556 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:09.556 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:09.557 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:09.635 [initandlisten] connection accepted from 127.0.0.5:43937 #209 (7 connections now open) m31001| Fri Jan 18 13:19:09.751 [conn209] end connection 127.0.0.5:43937 (6 connections now open) m31003| Fri Jan 18 13:19:09.752 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:09.752 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:09.752 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:09.752 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:09.752 [initandlisten] connection accepted from 127.0.0.5:33099 #210 (7 connections now open) m31001| Fri Jan 18 13:19:09.851 [conn210] end connection 127.0.0.5:33099 (6 connections now open) m31003| Fri Jan 18 13:19:09.905 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:09.905 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:09.905 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:09.905 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:09.905 [initandlisten] connection accepted from 127.0.0.5:59296 #211 (7 connections now open) m31001| Fri Jan 18 13:19:10.032 [conn211] end connection 127.0.0.5:59296 (6 connections now open) m31003| Fri Jan 18 13:19:10.072 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:10.072 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:10.072 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:10.072 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:10.126 [initandlisten] connection accepted from 127.0.0.5:36324 #212 (7 connections now open) m31003| Fri Jan 18 13:19:10.270 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:10.270 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:10.270 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:10.270 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:10.324 [conn212] end connection 127.0.0.5:36324 (6 connections now open) m31001| Fri Jan 18 13:19:10.367 [initandlisten] connection accepted from 127.0.0.5:49402 #213 (7 connections now open) m31003| Fri Jan 18 13:19:10.471 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:10.471 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:10.471 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:10.471 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:10.490 [conn213] end connection 127.0.0.5:49402 (6 connections now open) m31001| Fri Jan 18 13:19:10.519 [initandlisten] connection accepted from 127.0.0.5:57502 #214 (7 connections now open) m31003| Fri Jan 18 13:19:10.577 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:10.577 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:10.577 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:10.577 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:19:10.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Fri Jan 18 13:19:10.665 [conn214] end connection 127.0.0.5:57502 (6 connections now open) m31001| Fri Jan 18 13:19:10.692 [initandlisten] connection accepted from 127.0.0.5:58469 #215 (7 connections now open) m31003| Fri Jan 18 13:19:10.798 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:10.798 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:10.798 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:10.798 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:10.837 [conn215] end connection 127.0.0.5:58469 (6 connections now open) m31001| Fri Jan 18 13:19:10.892 [initandlisten] connection accepted from 127.0.0.5:58419 #216 (7 connections now open) m31003| Fri Jan 18 13:19:10.948 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:10.948 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:10.948 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:10.948 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:11.007 [conn216] end connection 127.0.0.5:58419 (6 connections now open) m31001| Fri Jan 18 13:19:11.014 [initandlisten] connection accepted from 127.0.0.5:36628 #217 (7 connections now open) m31003| Fri Jan 18 13:19:11.090 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:11.090 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:11.090 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:11.090 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:11.128 [conn217] end connection 127.0.0.5:36628 (6 connections now open) m31001| Fri Jan 18 13:19:11.131 [initandlisten] connection accepted from 127.0.0.5:37948 #218 (7 connections now open) m31001| Fri Jan 18 13:19:11.360 [conn218] end connection 127.0.0.5:37948 (6 connections now open) m31000| Fri Jan 18 13:19:11.365 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:380968 190ms m31003| Fri Jan 18 13:19:11.382 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:11.382 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:11.382 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:11.382 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:11.404 [initandlisten] connection accepted from 127.0.0.5:41047 #219 (7 connections now open) m31003| Fri Jan 18 13:19:11.459 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:11.459 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:11.459 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:11.459 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:11.520 [conn219] end connection 127.0.0.5:41047 (6 connections now open) m31001| Fri Jan 18 13:19:11.540 [initandlisten] connection accepted from 127.0.0.5:39532 #220 (7 connections now open) m31003| Fri Jan 18 13:19:11.652 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:11.652 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:11.652 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:11.652 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:11.701 [conn220] end connection 127.0.0.5:39532 (6 connections now open) m31001| Fri Jan 18 13:19:11.769 [initandlisten] connection accepted from 127.0.0.5:48657 #221 (7 connections now open) m31001| Fri Jan 18 13:19:11.906 [conn221] end connection 127.0.0.5:48657 (6 connections now open) m31003| Fri Jan 18 13:19:11.908 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:11.908 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:11.908 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:11.908 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:11.995 [initandlisten] connection accepted from 127.0.0.5:39211 #222 (7 connections now open) m31001| Fri Jan 18 13:19:12.119 [conn222] end connection 127.0.0.5:39211 (6 connections now open) m31003| Fri Jan 18 13:19:12.121 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:12.121 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:12.121 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:19:12.121 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31002| Fri Jan 18 13:19:12.205 [initandlisten] connection accepted from 127.0.0.5:54616 #100 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 225, "optime" : { "t" : 1358533152000, "i" : 3307 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 191, "optime" : { "t" : 1358533109000, "i" : 1076 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:12Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 191, "optime" : { "t" : 1358533109000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 191, "optime" : { "t" : 1358533151000, "i" : 1482 }, "optimeDate" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:12Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:12.353 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:12.353 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:12.353 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:19:12.353 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 190, "optime" : { "t" : 1358533152000, "i" : 3095 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 225, "optime" : { "t" : 1358533112000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:32Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 190, "optime" : { "t" : 1358533109000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 190, "optime" : { "t" : 1358533010000, "i" : 3944 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 190, "optime" : { "t" : 1358533152000, "i" : 3093 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 190, "optime" : { "t" : 1358533151000, "i" : 5 }, "optimeDate" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 190, "optime" : { "t" : 1358533109000, "i" : 1076 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:12Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 224, "optime" : { "t" : 1358533109000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 190, "optime" : { "t" : 1358533010000, "i" : 3344 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:12Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 190, "optime" : { "t" : 1358533150000, "i" : 7069 }, "optimeDate" : ISODate("2013-01-18T18:19:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:12Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 m31002| Fri Jan 18 13:19:12.415 [initandlisten] connection accepted from 127.0.0.5:46658 #101 (7 connections now open) m31002| Fri Jan 18 13:19:12.415 [initandlisten] connection accepted from 127.0.0.5:50060 #102 (8 connections now open) m31002| Fri Jan 18 13:19:12.415 [conn100] end connection 127.0.0.5:54616 (6 connections now open) m31001| Fri Jan 18 13:19:12.437 [conn192] end connection 127.0.0.5:52376 (5 connections now open) m31003| Fri Jan 18 13:19:12.543 [conn1] command admin.$cmd command: { replSetGetStatus: true } ntoreturn:1 keyUpdates:0 reslen:1149 189ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:12Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 191, "optime" : { "t" : 1358533109000, "i" : 1076 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:12Z"), "pingMs" : 38 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 191, "optime" : { "t" : 1358533109000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 37 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533010000, "i" : 4044 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 191, "optime" : { "t" : 1358533150000, "i" : 2974 }, "optimeDate" : ISODate("2013-01-18T18:19:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 190, "optime" : { "t" : 1358533152000, "i" : 3095 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 190, "optime" : { "t" : 1358533112000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:12Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 190, "optime" : { "t" : 1358533109000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 190, "optime" : { "t" : 1358533010000, "i" : 3944 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533152000, "i" : 3307 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "self" : true } ], "ok" : 1 } m31003| Fri Jan 18 13:19:12.548 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:12.548 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:12.548 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31002| Fri Jan 18 13:19:12.568 [conn101] end connection 127.0.0.5:46658 (6 connections now open) m31002| Fri Jan 18 13:19:12.631 [initandlisten] connection accepted from 127.0.0.5:32831 #103 (7 connections now open) m31002| Fri Jan 18 13:19:12.730 [conn102] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533010000|4044 } } cursorid:951108406489523 ntoreturn:0 keyUpdates:0 numYields: 65 locks(micros) r:321357 nreturned:175232 reslen:2978964 161ms m31003| Fri Jan 18 13:19:12.754 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:12.754 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:12.754 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:12.823 [initandlisten] connection accepted from 127.0.0.5:41714 #104 (8 connections now open) m31002| Fri Jan 18 13:19:12.824 [conn103] end connection 127.0.0.5:32831 (7 connections now open) m31003| Fri Jan 18 13:19:12.882 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:12.882 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:12.882 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:12.893 [conn104] end connection 127.0.0.5:41714 (6 connections now open) m31002| Fri Jan 18 13:19:12.912 [initandlisten] connection accepted from 127.0.0.5:43057 #105 (7 connections now open) m31002| Fri Jan 18 13:19:12.991 [conn105] end connection 127.0.0.5:43057 (6 connections now open) m31003| Fri Jan 18 13:19:13.054 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:13.054 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:13.054 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:13.104 [initandlisten] connection accepted from 127.0.0.5:38767 #106 (7 connections now open) m31003| Fri Jan 18 13:19:13.187 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:13.187 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:13.188 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:13.199 [initandlisten] connection accepted from 127.0.0.5:59185 #107 (8 connections now open) m31002| Fri Jan 18 13:19:13.200 [conn106] end connection 127.0.0.5:38767 (6 connections now open) m31003| Fri Jan 18 13:19:13.385 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:13.385 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:13.385 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:13.451 [conn107] end connection 127.0.0.5:59185 (6 connections now open) m31002| Fri Jan 18 13:19:13.474 [initandlisten] connection accepted from 127.0.0.5:52461 #108 (7 connections now open) m31003| Fri Jan 18 13:19:13.651 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:13.651 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:13.651 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:13.744 [initandlisten] connection accepted from 127.0.0.5:46801 #109 (8 connections now open) m31002| Fri Jan 18 13:19:13.744 [conn108] end connection 127.0.0.5:52461 (6 connections now open) m31003| Fri Jan 18 13:19:13.860 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:13.860 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:13.860 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:13.933 [initandlisten] connection accepted from 127.0.0.5:32916 #110 (8 connections now open) m31002| Fri Jan 18 13:19:13.933 [conn109] end connection 127.0.0.5:46801 (6 connections now open) m31002| Fri Jan 18 13:19:14.153 [conn110] end connection 127.0.0.5:32916 (6 connections now open) m31003| Fri Jan 18 13:19:14.194 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:14.194 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:14.194 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:14.194 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:19:14.194 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31001| Fri Jan 18 13:19:14.256 [initandlisten] connection accepted from 127.0.0.5:58373 #223 (6 connections now open) m31001| Fri Jan 18 13:19:14.377 [conn223] end connection 127.0.0.5:58373 (5 connections now open) m31003| Fri Jan 18 13:19:14.380 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.3:31001 m31002| Fri Jan 18 13:19:14.380 [conn102] end connection 127.0.0.5:50060 (5 connections now open) m31001| Fri Jan 18 13:19:14.380 [initandlisten] connection accepted from 127.0.0.5:39633 #224 (6 connections now open) m31003| Fri Jan 18 13:19:14.380 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:14.380 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:14.380 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:14.380 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:14.480 [initandlisten] connection accepted from 127.0.0.5:52867 #225 (7 connections now open) m31001| Fri Jan 18 13:19:14.665 [conn225] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533010000|5044 } } cursorid:962919031960650 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 727 locks(micros) r:126935 nreturned:101 reslen:10423 133ms m31003| Fri Jan 18 13:19:14.669 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:14.669 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:14.669 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:14.669 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:14.745 [initandlisten] connection accepted from 127.0.0.5:34503 #226 (8 connections now open) m31001| Fri Jan 18 13:19:14.745 [conn225] end connection 127.0.0.5:52867 (6 connections now open) m31001| Fri Jan 18 13:19:14.912 [conn224] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533010000|5044 } } cursorid:962620083305950 ntoreturn:0 keyUpdates:0 numYields: 1269 locks(micros) r:301325 nreturned:190918 reslen:3245626 166ms m31003| Fri Jan 18 13:19:14.950 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:14.950 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:14.950 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:14.950 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:14.962 [conn226] end connection 127.0.0.5:34503 (6 connections now open) m31001| Fri Jan 18 13:19:15.024 [initandlisten] connection accepted from 127.0.0.5:36644 #227 (7 connections now open) m31003| Fri Jan 18 13:19:15.162 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:15.162 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:15.162 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:15.162 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:15.194 [conn227] end connection 127.0.0.5:36644 (6 connections now open) m31001| Fri Jan 18 13:19:15.220 [initandlisten] connection accepted from 127.0.0.5:45891 #228 (7 connections now open) m31003| Fri Jan 18 13:19:15.299 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:15.299 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:15.299 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:15.299 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:15.329 [conn228] end connection 127.0.0.5:45891 (6 connections now open) m31001| Fri Jan 18 13:19:15.363 [initandlisten] connection accepted from 127.0.0.5:37559 #229 (7 connections now open) m31003| Fri Jan 18 13:19:15.533 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:15.533 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:15.533 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:15.533 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:15.585 [conn229] end connection 127.0.0.5:37559 (6 connections now open) m31001| Fri Jan 18 13:19:15.635 [initandlisten] connection accepted from 127.0.0.5:57918 #230 (7 connections now open) m31003| Fri Jan 18 13:19:15.807 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:15.807 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:15.807 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:15.807 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:15.869 [initandlisten] connection accepted from 127.0.0.5:59697 #231 (8 connections now open) m31001| Fri Jan 18 13:19:15.870 [conn230] end connection 127.0.0.5:57918 (7 connections now open) m31003| Fri Jan 18 13:19:15.970 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:15.970 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:15.970 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:15.970 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:16.003 [conn231] end connection 127.0.0.5:59697 (6 connections now open) m31001| Fri Jan 18 13:19:16.013 [initandlisten] connection accepted from 127.0.0.5:53804 #232 (7 connections now open) m31003| Fri Jan 18 13:19:16.091 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:16.091 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:16.091 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:16.091 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:16.103 [initandlisten] connection accepted from 127.0.0.5:40028 #233 (8 connections now open) m31001| Fri Jan 18 13:19:16.103 [conn232] end connection 127.0.0.5:53804 (7 connections now open) m31001| Fri Jan 18 13:19:16.232 [conn233] end connection 127.0.0.5:40028 (6 connections now open) m31003| Fri Jan 18 13:19:16.236 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:16.236 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:16.236 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:16.236 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:16.326 [initandlisten] connection accepted from 127.0.0.5:45369 #234 (7 connections now open) m31003| Fri Jan 18 13:19:16.451 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:16.451 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:16.451 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:16.451 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:16.538 [initandlisten] connection accepted from 127.0.0.5:41281 #235 (8 connections now open) m31001| Fri Jan 18 13:19:16.538 [conn234] end connection 127.0.0.5:45369 (7 connections now open) m31003| Fri Jan 18 13:19:16.653 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:16.653 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:16.653 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:16.653 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:16.708 [conn235] end connection 127.0.0.5:41281 (6 connections now open) m31001| Fri Jan 18 13:19:16.792 [initandlisten] connection accepted from 127.0.0.5:44891 #236 (7 connections now open) m31003| Fri Jan 18 13:19:16.934 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:16.934 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:16.934 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:16.934 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:16.991 [conn236] end connection 127.0.0.5:44891 (6 connections now open) m31001| Fri Jan 18 13:19:17.065 [initandlisten] connection accepted from 127.0.0.5:45858 #237 (7 connections now open) m31003| Fri Jan 18 13:19:17.223 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:17.223 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:17.223 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:17.223 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:17.270 [conn237] end connection 127.0.0.5:45858 (6 connections now open) m31001| Fri Jan 18 13:19:17.308 [initandlisten] connection accepted from 127.0.0.5:42753 #238 (7 connections now open) m31003| Fri Jan 18 13:19:17.419 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:17.419 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:17.419 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:17.419 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:17.493 [initandlisten] connection accepted from 127.0.0.5:38618 #239 (8 connections now open) m31001| Fri Jan 18 13:19:17.493 [conn238] end connection 127.0.0.5:42753 (7 connections now open) m31003| Fri Jan 18 13:19:17.556 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:17.556 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:17.556 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:17.556 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:17.612 [conn239] end connection 127.0.0.5:38618 (6 connections now open) m31001| Fri Jan 18 13:19:17.657 [initandlisten] connection accepted from 127.0.0.5:47820 #240 (7 connections now open) m31003| Fri Jan 18 13:19:17.755 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:17.756 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:17.756 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:17.756 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:17.792 [conn240] end connection 127.0.0.5:47820 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 230, "optime" : { "t" : 1358533157000, "i" : 7366 }, "optimeDate" : ISODate("2013-01-18T18:19:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 196, "optime" : { "t" : 1358533115000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 196, "optime" : { "t" : 1358533115000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 196, "optime" : { "t" : 1358533157000, "i" : 3392 }, "optimeDate" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 195, "optime" : { "t" : 1358533156000, "i" : 3225 }, "optimeDate" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:17Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 230, "optime" : { "t" : 1358533115000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 195, "optime" : { "t" : 1358533115000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:17Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 195, "optime" : { "t" : 1358533010000, "i" : 5944 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 195, "optime" : { "t" : 1358533156000, "i" : 3221 }, "optimeDate" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 195, "optime" : { "t" : 1358533157000, "i" : 50 }, "optimeDate" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:17Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 195, "optime" : { "t" : 1358533115000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 229, "optime" : { "t" : 1358533115000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 195, "optime" : { "t" : 1358533010000, "i" : 6244 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 195, "optime" : { "t" : 1358533156000, "i" : 8905 }, "optimeDate" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:17Z"), "myState" : 2, "syncingTo" : "127.0.0.3:31001", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 196, "optime" : { "t" : 1358533115000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:16Z"), "pingMs" : 29 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 196, "optime" : { "t" : 1358533115000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:17Z"), "pingMs" : 30 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533010000, "i" : 6644 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "errmsg" : "syncing to: 127.0.0.3:31001", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 196, "optime" : { "t" : 1358533155000, "i" : 438 }, "optimeDate" : ISODate("2013-01-18T18:19:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 195, "optime" : { "t" : 1358533156000, "i" : 3225 }, "optimeDate" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:17Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 195, "optime" : { "t" : 1358533115000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:16Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 195, "optime" : { "t" : 1358533115000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:17Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 195, "optime" : { "t" : 1358533010000, "i" : 5944 }, "optimeDate" : ISODate("2013-01-18T18:16:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533157000, "i" : 7366 }, "optimeDate" : ISODate("2013-01-18T18:19:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.3:31001" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:19:17.918 [initandlisten] connection accepted from 127.0.0.5:58498 #241 (7 connections now open) m31003| Fri Jan 18 13:19:18.061 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:18.061 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:18.061 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:18.061 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:18.100 [conn241] end connection 127.0.0.5:58498 (6 connections now open) m31001| Fri Jan 18 13:19:18.166 [initandlisten] connection accepted from 127.0.0.5:44183 #242 (7 connections now open) m31003| Fri Jan 18 13:19:18.227 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:18.227 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:18.227 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:18.227 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:18.300 [conn242] end connection 127.0.0.5:44183 (6 connections now open) m31001| Fri Jan 18 13:19:18.300 [initandlisten] connection accepted from 127.0.0.5:46016 #243 (8 connections now open) m31003| Fri Jan 18 13:19:18.517 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:18.517 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:18.517 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:18.517 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:18.607 [conn243] end connection 127.0.0.5:46016 (6 connections now open) m31001| Fri Jan 18 13:19:18.607 [initandlisten] connection accepted from 127.0.0.5:58384 #244 (8 connections now open) m31003| Fri Jan 18 13:19:18.705 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:18.706 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:18.706 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:18.706 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:18.706 [conn244] end connection 127.0.0.5:58384 (6 connections now open) m31001| Fri Jan 18 13:19:18.784 [initandlisten] connection accepted from 127.0.0.5:47640 #245 (7 connections now open) m31003| Fri Jan 18 13:19:18.967 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:18.967 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:18.967 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:18.967 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:19.043 [conn245] end connection 127.0.0.5:47640 (6 connections now open) m31001| Fri Jan 18 13:19:19.108 [initandlisten] connection accepted from 127.0.0.5:42807 #246 (7 connections now open) m31003| Fri Jan 18 13:19:19.226 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:19.226 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:19.226 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:19.226 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:19.298 [conn246] end connection 127.0.0.5:42807 (6 connections now open) m31001| Fri Jan 18 13:19:19.359 [initandlisten] connection accepted from 127.0.0.5:46056 #247 (7 connections now open) m31003| Fri Jan 18 13:19:19.488 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:19.488 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:19.488 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:19.488 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:19.582 [initandlisten] connection accepted from 127.0.0.5:55395 #248 (8 connections now open) m31001| Fri Jan 18 13:19:19.582 [conn247] end connection 127.0.0.5:46056 (7 connections now open) m31001| Fri Jan 18 13:19:19.714 [conn248] end connection 127.0.0.5:55395 (6 connections now open) m31003| Fri Jan 18 13:19:19.717 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:19.717 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:19.717 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:19.717 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:19.750 [initandlisten] connection accepted from 127.0.0.5:33144 #249 (7 connections now open) m31003| Fri Jan 18 13:19:19.890 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:19.890 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:19.890 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:19.890 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:19.931 [conn249] end connection 127.0.0.5:33144 (6 connections now open) m31001| Fri Jan 18 13:19:19.985 [initandlisten] connection accepted from 127.0.0.5:59215 #250 (7 connections now open) m31003| Fri Jan 18 13:19:20.174 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:20.174 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:20.174 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:20.174 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:20.255 [conn250] end connection 127.0.0.5:59215 (6 connections now open) m31001| Fri Jan 18 13:19:20.255 [initandlisten] connection accepted from 127.0.0.5:58669 #251 (8 connections now open) m31002| Fri Jan 18 13:19:20.313 [conn64] end connection 127.0.0.6:45666 (4 connections now open) m31002| Fri Jan 18 13:19:20.313 [initandlisten] connection accepted from 127.0.0.6:39608 #111 (5 connections now open) m31003| Fri Jan 18 13:19:20.336 [conn26] end connection 127.0.0.3:46663 (3 connections now open) m31003| Fri Jan 18 13:19:20.336 [initandlisten] connection accepted from 127.0.0.3:38716 #29 (4 connections now open) m31001| Fri Jan 18 13:19:20.399 [conn251] end connection 127.0.0.5:58669 (6 connections now open) m31003| Fri Jan 18 13:19:20.401 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:20.401 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:20.401 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:20.401 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:20.578 [initandlisten] connection accepted from 127.0.0.5:55606 #252 (7 connections now open) m31003| Fri Jan 18 13:19:20.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Fri Jan 18 13:19:20.705 [conn252] end connection 127.0.0.5:55606 (6 connections now open) m31003| Fri Jan 18 13:19:20.709 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:20.709 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:20.709 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:20.709 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:20.709 [initandlisten] connection accepted from 127.0.0.5:42698 #253 (7 connections now open) m31003| Fri Jan 18 13:19:20.948 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:20.948 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:20.948 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:20.949 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:19:21.005 [conn27] end connection 127.0.0.4:52783 (3 connections now open) m31003| Fri Jan 18 13:19:21.005 [initandlisten] connection accepted from 127.0.0.4:48346 #30 (4 connections now open) m31001| Fri Jan 18 13:19:21.038 [initandlisten] connection accepted from 127.0.0.5:51790 #254 (8 connections now open) m31001| Fri Jan 18 13:19:21.038 [conn253] end connection 127.0.0.5:42698 (7 connections now open) m31003| Fri Jan 18 13:19:21.159 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:21.159 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:21.159 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:21.159 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:21.166 [conn254] end connection 127.0.0.5:51790 (6 connections now open) m31001| Fri Jan 18 13:19:21.234 [initandlisten] connection accepted from 127.0.0.5:43212 #255 (7 connections now open) m31001| Fri Jan 18 13:19:21.356 [conn142] end connection 127.0.0.2:47053 (6 connections now open) m31001| Fri Jan 18 13:19:21.356 [initandlisten] connection accepted from 127.0.0.2:38277 #256 (7 connections now open) m31003| Fri Jan 18 13:19:21.376 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:21.376 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:21.376 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:21.376 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:21.459 [initandlisten] connection accepted from 127.0.0.5:40786 #257 (8 connections now open) m31001| Fri Jan 18 13:19:21.459 [conn255] end connection 127.0.0.5:43212 (6 connections now open) m31000| Fri Jan 18 13:19:21.464 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.2, filling with zeroes... m31003| Fri Jan 18 13:19:21.607 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:21.607 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:21.607 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:21.607 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:21.690 [conn257] end connection 127.0.0.5:40786 (6 connections now open) m31001| Fri Jan 18 13:19:21.717 [initandlisten] connection accepted from 127.0.0.5:44947 #258 (7 connections now open) m31003| Fri Jan 18 13:19:21.879 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:21.879 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:21.879 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:21.879 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:21.881 [conn258] end connection 127.0.0.5:44947 (6 connections now open) m31001| Fri Jan 18 13:19:21.938 [initandlisten] connection accepted from 127.0.0.5:50281 #259 (7 connections now open) m31000| Fri Jan 18 13:19:22.073 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.2, size: 64MB, took 0.609 secs m31000| Fri Jan 18 13:19:22.073 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:610065 610ms m31004| Fri Jan 18 13:19:22.074 [FileAllocator] allocating new datafile /data/db/testReplSet-4/foo.2, filling with zeroes... m31003| Fri Jan 18 13:19:22.077 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:22.077 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:22.077 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:19:22.077 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31001| Fri Jan 18 13:19:22.104 [conn259] end connection 127.0.0.5:50281 (6 connections now open) m31002| Fri Jan 18 13:19:22.146 [initandlisten] connection accepted from 127.0.0.5:40154 #112 (6 connections now open) m31002| Fri Jan 18 13:19:22.146 [conn65] end connection 127.0.0.5:35698 (5 connections now open) m31002| Fri Jan 18 13:19:22.243 [initandlisten] connection accepted from 127.0.0.5:35530 #113 (6 connections now open) m31000| Fri Jan 18 13:19:22.313 [conn48] end connection 127.0.0.6:43398 (11 connections now open) m31000| Fri Jan 18 13:19:22.314 [initandlisten] connection accepted from 127.0.0.6:36505 #51 (12 connections now open) m31003| Fri Jan 18 13:19:22.316 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:22.316 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 m31003| Fri Jan 18 13:19:22.316 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:22.316 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:22.316 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:19:22.316 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31000| Fri Jan 18 13:19:22.337 [conn49] end connection 127.0.0.3:52472 (11 connections now open) m31000| Fri Jan 18 13:19:22.337 [initandlisten] connection accepted from 127.0.0.3:49130 #52 (13 connections now open) m31002| Fri Jan 18 13:19:22.370 [conn112] end connection 127.0.0.5:40154 (5 connections now open) m31002| Fri Jan 18 13:19:22.373 [initandlisten] connection accepted from 127.0.0.5:51313 #114 (6 connections now open) m31001| Fri Jan 18 13:19:22.414 [initandlisten] connection accepted from 127.0.0.5:42695 #260 (7 connections now open) m31001| Fri Jan 18 13:19:22.414 [conn224] end connection 127.0.0.5:39633 (5 connections now open) m31004| Fri Jan 18 13:19:22.608 [FileAllocator] done allocating datafile /data/db/testReplSet-4/foo.2, size: 64MB, took 0.533 secs m31001| Fri Jan 18 13:19:22.626 [conn260] end connection 127.0.0.5:42695 (5 connections now open) m31003| Fri Jan 18 13:19:22.628 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:22.628 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.3:31001 m31003| Fri Jan 18 13:19:22.628 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:22.628 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:22.628 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31002| Fri Jan 18 13:19:22.704 [conn114] end connection 127.0.0.5:51313 (5 connections now open) m31001| Fri Jan 18 13:19:22.720 [initandlisten] connection accepted from 127.0.0.5:46299 #261 (6 connections now open) m31001| Fri Jan 18 13:19:22.720 [initandlisten] connection accepted from 127.0.0.5:54606 #262 (7 connections now open) m31003| Fri Jan 18 13:19:22.824 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:22.824 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:22.824 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:22.824 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 STATUS FROM : 127.0.0.2:31000 m31001| Fri Jan 18 13:19:22.904 [conn261] end connection 127.0.0.5:46299 (6 connections now open) m31001| Fri Jan 18 13:19:22.927 [initandlisten] connection accepted from 127.0.0.5:47331 #263 (7 connections now open) m31000| Fri Jan 18 13:19:23.005 [conn50] end connection 127.0.0.4:42807 (11 connections now open) m31000| Fri Jan 18 13:19:23.005 [initandlisten] connection accepted from 127.0.0.4:49145 #53 (12 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 236, "optime" : { "t" : 1358533163000, "i" : 46 }, "optimeDate" : ISODate("2013-01-18T18:19:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 202, "optime" : { "t" : 1358533120000, "i" : 1412 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 202, "optime" : { "t" : 1358533120000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:23Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 202, "optime" : { "t" : 1358533161000, "i" : 3020 }, "optimeDate" : ISODate("2013-01-18T18:19:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 201, "optime" : { "t" : 1358533162000, "i" : 4090 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 236, "optime" : { "t" : 1358533120000, "i" : 1412 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533120000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:23Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533011000, "i" : 300 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533161000, "i" : 4004 }, "optimeDate" : ISODate("2013-01-18T18:19:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 201, "optime" : { "t" : 1358533162000, "i" : 14614 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533120000, "i" : 1412 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 235, "optime" : { "t" : 1358533120000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533011000, "i" : 500 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533162000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 m31001| Fri Jan 18 13:19:23.094 [conn262] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|400 } } cursorid:997618597882269 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:363564 nreturned:209266 reslen:3557542 189ms m31003| Fri Jan 18 13:19:23.134 [conn1] command admin.$cmd command: { replSetGetStatus: true } ntoreturn:1 keyUpdates:0 reslen:1149 101ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:23Z"), "myState" : 2, "syncingTo" : "127.0.0.3:31001", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 202, "optime" : { "t" : 1358533120000, "i" : 1412 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:22Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 202, "optime" : { "t" : 1358533120000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:23Z"), "pingMs" : 39 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358533011000, "i" : 500 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "errmsg" : "syncing to: 127.0.0.3:31001", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 202, "optime" : { "t" : 1358533157000, "i" : 7366 }, "optimeDate" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:23.136 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:23.136 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:23.136 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:23.136 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 201, "optime" : { "t" : 1358533162000, "i" : 3429 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533120000, "i" : 1412 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533120000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:23Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358533011000, "i" : 200 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:23Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358533162000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.3:31001" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:19:23.196 [conn263] end connection 127.0.0.5:47331 (6 connections now open) m31001| Fri Jan 18 13:19:23.313 [initandlisten] connection accepted from 127.0.0.5:39968 #264 (7 connections now open) m31003| Fri Jan 18 13:19:23.454 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:23.454 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:23.454 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:23.454 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:23.467 [initandlisten] connection accepted from 127.0.0.5:53242 #265 (8 connections now open) m31001| Fri Jan 18 13:19:23.467 [conn264] end connection 127.0.0.5:39968 (6 connections now open) m31001| Fri Jan 18 13:19:23.641 [conn265] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|700 } } cursorid:1001097517031371 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 824 locks(micros) r:127382 nreturned:101 reslen:10423 132ms m31001| Fri Jan 18 13:19:23.665 [conn265] end connection 127.0.0.5:53242 (6 connections now open) m31003| Fri Jan 18 13:19:23.689 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:23.689 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:23.689 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:23.689 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:23.689 [initandlisten] connection accepted from 127.0.0.5:50937 #266 (7 connections now open) m31001| Fri Jan 18 13:19:23.751 [conn266] end connection 127.0.0.5:50937 (6 connections now open) m31003| Fri Jan 18 13:19:23.755 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:23.755 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:23.755 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:23.755 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:23.792 [initandlisten] connection accepted from 127.0.0.5:58256 #267 (7 connections now open) m31003| Fri Jan 18 13:19:23.945 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:23.945 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:23.945 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:23.945 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:23.960 [conn267] end connection 127.0.0.5:58256 (6 connections now open) m31001| Fri Jan 18 13:19:24.046 [initandlisten] connection accepted from 127.0.0.5:42197 #268 (7 connections now open) m31003| Fri Jan 18 13:19:24.162 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:24.162 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:24.162 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:24.162 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:24.248 [conn268] end connection 127.0.0.5:42197 (6 connections now open) m31001| Fri Jan 18 13:19:24.327 [initandlisten] connection accepted from 127.0.0.5:35231 #269 (7 connections now open) m31003| Fri Jan 18 13:19:24.448 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:24.448 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:24.448 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:19:24.448 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31001| Fri Jan 18 13:19:24.530 [conn269] end connection 127.0.0.5:35231 (6 connections now open) m31002| Fri Jan 18 13:19:24.548 [initandlisten] connection accepted from 127.0.0.5:60310 #115 (6 connections now open) m31003| Fri Jan 18 13:19:24.767 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:24.767 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 m31003| Fri Jan 18 13:19:24.767 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:24.767 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31001| Fri Jan 18 13:19:24.805 [conn262] end connection 127.0.0.5:54606 (5 connections now open) m31002| Fri Jan 18 13:19:24.858 [conn115] end connection 127.0.0.5:60310 (5 connections now open) m31002| Fri Jan 18 13:19:24.945 [initandlisten] connection accepted from 127.0.0.5:45682 #116 (6 connections now open) m31002| Fri Jan 18 13:19:24.945 [initandlisten] connection accepted from 127.0.0.5:54379 #117 (7 connections now open) m31002| Fri Jan 18 13:19:25.068 [conn117] end connection 127.0.0.5:54379 (6 connections now open) m31003| Fri Jan 18 13:19:25.068 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:25.068 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:25.068 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:25.069 [initandlisten] connection accepted from 127.0.0.5:57478 #118 (7 connections now open) m31003| Fri Jan 18 13:19:25.214 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:25.214 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:25.214 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:25.217 [conn118] end connection 127.0.0.5:57478 (6 connections now open) m31002| Fri Jan 18 13:19:25.261 [conn116] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|1300 } } cursorid:1004675799929368 ntoreturn:0 keyUpdates:0 numYields: 3 locks(micros) r:334757 nreturned:220203 reslen:3743471 193ms m31002| Fri Jan 18 13:19:25.316 [initandlisten] connection accepted from 127.0.0.5:37204 #119 (7 connections now open) m31003| Fri Jan 18 13:19:25.486 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:25.486 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:25.486 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:25.523 [initandlisten] connection accepted from 127.0.0.5:41986 #120 (8 connections now open) m31002| Fri Jan 18 13:19:25.548 [conn119] end connection 127.0.0.5:37204 (7 connections now open) m31003| Fri Jan 18 13:19:25.702 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:25.702 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:25.702 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:25.734 [conn120] end connection 127.0.0.5:41986 (6 connections now open) m31002| Fri Jan 18 13:19:25.738 [initandlisten] connection accepted from 127.0.0.5:37817 #121 (7 connections now open) m31003| Fri Jan 18 13:19:25.811 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:25.811 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:25.811 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:25.824 [conn121] end connection 127.0.0.5:37817 (6 connections now open) m31002| Fri Jan 18 13:19:25.911 [initandlisten] connection accepted from 127.0.0.5:47887 #122 (7 connections now open) m31003| Fri Jan 18 13:19:26.044 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:26.044 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:26.044 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:26.078 [initandlisten] connection accepted from 127.0.0.5:38995 #123 (8 connections now open) m31002| Fri Jan 18 13:19:26.078 [conn122] end connection 127.0.0.5:47887 (7 connections now open) m31002| Fri Jan 18 13:19:26.257 [conn123] end connection 127.0.0.5:38995 (6 connections now open) m31003| Fri Jan 18 13:19:26.295 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:26.295 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:26.295 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:26.464 [initandlisten] connection accepted from 127.0.0.5:47612 #124 (7 connections now open) m31003| Fri Jan 18 13:19:26.615 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:26.615 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:26.615 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:26.615 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:19:26.615 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31001| Fri Jan 18 13:19:26.615 [initandlisten] connection accepted from 127.0.0.5:59402 #270 (6 connections now open) m31002| Fri Jan 18 13:19:26.674 [conn124] end connection 127.0.0.5:47612 (6 connections now open) m31003| Fri Jan 18 13:19:26.790 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.3:31001 m31003| Fri Jan 18 13:19:26.790 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:26.790 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:26.790 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:26.790 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:26.799 [conn270] end connection 127.0.0.5:59402 (5 connections now open) m31002| Fri Jan 18 13:19:26.841 [conn116] end connection 127.0.0.5:45682 (5 connections now open) m31001| Fri Jan 18 13:19:26.846 [initandlisten] connection accepted from 127.0.0.5:51755 #271 (6 connections now open) m31001| Fri Jan 18 13:19:26.846 [initandlisten] connection accepted from 127.0.0.5:50100 #272 (7 connections now open) m31003| Fri Jan 18 13:19:26.927 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:26.927 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:26.927 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:26.927 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:26.974 [initandlisten] connection accepted from 127.0.0.5:35917 #273 (8 connections now open) m31001| Fri Jan 18 13:19:26.975 [conn272] end connection 127.0.0.5:50100 (7 connections now open) m31003| Fri Jan 18 13:19:27.157 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:27.157 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:27.157 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:27.157 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:27.189 [conn273] end connection 127.0.0.5:35917 (6 connections now open) m31001| Fri Jan 18 13:19:27.189 [initandlisten] connection accepted from 127.0.0.5:36666 #274 (8 connections now open) m31001| Fri Jan 18 13:19:27.227 [conn271] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|2200 } } cursorid:1015438502969514 ntoreturn:0 keyUpdates:0 numYields: 4 locks(micros) r:317144 nreturned:228932 reslen:3891864 196ms m31001| Fri Jan 18 13:19:27.411 [conn274] end connection 127.0.0.5:36666 (6 connections now open) m31000| Fri Jan 18 13:19:27.970 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:1500407 750ms m31003| Fri Jan 18 13:19:27.971 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:27.971 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:27.971 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:27.971 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:28.026 [initandlisten] connection accepted from 127.0.0.5:58578 #275 (7 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:19:28.191 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:234305 117ms m31003| Fri Jan 18 13:19:28.233 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:28.233 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:28.233 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:28.233 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:28.317 [conn275] end connection 127.0.0.5:58578 (6 connections now open) m31001| Fri Jan 18 13:19:28.327 [initandlisten] connection accepted from 127.0.0.5:35884 #276 (7 connections now open) m31000| Fri Jan 18 13:19:28.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31001| Fri Jan 18 13:19:28.561 [conn276] end connection 127.0.0.5:35884 (6 connections now open) m31003| Fri Jan 18 13:19:28.661 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:28.661 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:28.661 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:19:28.661 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31002| Fri Jan 18 13:19:28.661 [initandlisten] connection accepted from 127.0.0.5:44154 #125 (6 connections now open) m31002| Fri Jan 18 13:19:28.873 [conn125] end connection 127.0.0.5:44154 (5 connections now open) m31003| Fri Jan 18 13:19:28.919 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 m31003| Fri Jan 18 13:19:28.919 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:28.919 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:28.919 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31001| Fri Jan 18 13:19:28.919 [conn271] end connection 127.0.0.5:51755 (5 connections now open) m31002| Fri Jan 18 13:19:28.928 [initandlisten] connection accepted from 127.0.0.5:47713 #126 (6 connections now open) m31002| Fri Jan 18 13:19:28.928 [initandlisten] connection accepted from 127.0.0.5:39396 #127 (7 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 241, "optime" : { "t" : 1358533168000, "i" : 10129 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533125000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:28Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533125000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533167000, "i" : 1771 }, "optimeDate" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:28Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 206, "optime" : { "t" : 1358533168000, "i" : 4616 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 241, "optime" : { "t" : 1358533128000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:48Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 206, "optime" : { "t" : 1358533125000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 206, "optime" : { "t" : 1358533011000, "i" : 2600 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 206, "optime" : { "t" : 1358533168000, "i" : 4570 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 207, "optime" : { "t" : 1358533167000, "i" : 21 }, "optimeDate" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533125000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:28Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 241, "optime" : { "t" : 1358533128000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:48Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533011000, "i" : 2300 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:28Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533167000, "i" : 4 }, "optimeDate" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:28Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 m31002| Fri Jan 18 13:19:29.085 [conn127] end connection 127.0.0.5:39396 (6 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:29Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:18:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 208, "optime" : { "t" : 1358533128000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:28Z"), "pingMs" : 42 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 208, "optime" : { "t" : 1358533125000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 35 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 240, "optime" : { "t" : 1358533011000, "i" : 2800 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 208, "optime" : { "t" : 1358533168000, "i" : 1871 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:29.088 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:29.088 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:29.088 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 207, "optime" : { "t" : 1358533168000, "i" : 4616 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533128000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:28Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533125000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:29Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 207, "optime" : { "t" : 1358533011000, "i" : 2600 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 240, "optime" : { "t" : 1358533168000, "i" : 10129 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31002| Fri Jan 18 13:19:29.175 [initandlisten] connection accepted from 127.0.0.5:35840 #128 (7 connections now open) m31003| Fri Jan 18 13:19:29.254 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:29.254 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:29.254 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:29.323 [conn126] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|2800 } } cursorid:1021809093090104 ntoreturn:0 keyUpdates:0 numYields: 2 locks(micros) r:341558 nreturned:228701 reslen:3887937 212ms m31002| Fri Jan 18 13:19:29.336 [conn128] end connection 127.0.0.5:35840 (6 connections now open) m31002| Fri Jan 18 13:19:29.358 [initandlisten] connection accepted from 127.0.0.5:38383 #129 (7 connections now open) m31003| Fri Jan 18 13:19:29.439 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:29.439 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:29.440 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:29.452 [conn129] end connection 127.0.0.5:38383 (6 connections now open) m31002| Fri Jan 18 13:19:29.525 [initandlisten] connection accepted from 127.0.0.5:36499 #130 (7 connections now open) m31003| Fri Jan 18 13:19:29.683 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:29.683 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:29.683 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:29.757 [initandlisten] connection accepted from 127.0.0.5:51002 #131 (8 connections now open) m31002| Fri Jan 18 13:19:29.768 [conn130] end connection 127.0.0.5:36499 (7 connections now open) m31003| Fri Jan 18 13:19:29.950 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:29.950 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:29.950 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:29.997 [conn131] end connection 127.0.0.5:51002 (6 connections now open) m31002| Fri Jan 18 13:19:29.997 [initandlisten] connection accepted from 127.0.0.5:60229 #132 (8 connections now open) m31002| Fri Jan 18 13:19:30.113 [conn132] end connection 127.0.0.5:60229 (6 connections now open) m31003| Fri Jan 18 13:19:30.454 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:30.454 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:30.454 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:30.483 [initandlisten] connection accepted from 127.0.0.5:38040 #133 (7 connections now open) m31002| Fri Jan 18 13:19:30.648 [conn133] end connection 127.0.0.5:38040 (6 connections now open) m31003| Fri Jan 18 13:19:30.688 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:30.688 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:30.688 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:30.779 [initandlisten] connection accepted from 127.0.0.5:32989 #134 (7 connections now open) m31003| Fri Jan 18 13:19:30.930 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:30.930 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:30.930 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:31.019 [conn134] end connection 127.0.0.5:32989 (6 connections now open) m31002| Fri Jan 18 13:19:31.019 [initandlisten] connection accepted from 127.0.0.5:58728 #135 (8 connections now open) m31003| Fri Jan 18 13:19:31.141 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:31.141 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:31.141 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:31.148 [conn135] end connection 127.0.0.5:58728 (6 connections now open) m31002| Fri Jan 18 13:19:31.178 [initandlisten] connection accepted from 127.0.0.5:59043 #136 (7 connections now open) m31002| Fri Jan 18 13:19:31.327 [conn136] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|3700 } } cursorid:1031330310742339 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 938 locks(micros) r:133420 nreturned:101 reslen:10423 110ms m31003| Fri Jan 18 13:19:31.330 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:31.330 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:31.330 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:31.398 [conn136] end connection 127.0.0.5:59043 (6 connections now open) m31002| Fri Jan 18 13:19:31.399 [initandlisten] connection accepted from 127.0.0.5:55396 #137 (8 connections now open) m31003| Fri Jan 18 13:19:31.556 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:31.556 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:31.556 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:31.584 [conn137] end connection 127.0.0.5:55396 (6 connections now open) m31002| Fri Jan 18 13:19:31.671 [initandlisten] connection accepted from 127.0.0.5:60088 #138 (7 connections now open) m31003| Fri Jan 18 13:19:31.828 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:31.828 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:31.828 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:31.868 [initandlisten] connection accepted from 127.0.0.5:49911 #139 (8 connections now open) m31002| Fri Jan 18 13:19:31.868 [conn138] end connection 127.0.0.5:60088 (7 connections now open) m31003| Fri Jan 18 13:19:32.077 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:32.077 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:32.077 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:32.151 [conn139] end connection 127.0.0.5:49911 (6 connections now open) m31002| Fri Jan 18 13:19:32.207 [initandlisten] connection accepted from 127.0.0.5:41291 #140 (7 connections now open) m31003| Fri Jan 18 13:19:32.347 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:32.347 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:32.347 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:32.347 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:19:32.347 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31001| Fri Jan 18 13:19:32.402 [initandlisten] connection accepted from 127.0.0.5:37308 #277 (6 connections now open) m31002| Fri Jan 18 13:19:32.442 [conn140] end connection 127.0.0.5:41291 (6 connections now open) m31003| Fri Jan 18 13:19:32.633 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.3:31001 m31001| Fri Jan 18 13:19:32.673 [conn277] end connection 127.0.0.5:37308 (5 connections now open) m31001| Fri Jan 18 13:19:32.683 [initandlisten] connection accepted from 127.0.0.5:58538 #278 (6 connections now open) m31002| Fri Jan 18 13:19:32.702 [conn126] end connection 127.0.0.5:47713 (5 connections now open) m31003| Fri Jan 18 13:19:32.722 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:32.722 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:32.722 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:32.722 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:32.776 [initandlisten] connection accepted from 127.0.0.5:58285 #279 (7 connections now open) m31003| Fri Jan 18 13:19:32.936 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:32.936 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:32.936 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:32.936 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:32.962 [initandlisten] connection accepted from 127.0.0.5:46146 #280 (8 connections now open) m31001| Fri Jan 18 13:19:32.962 [conn279] end connection 127.0.0.5:58285 (7 connections now open) m31003| Fri Jan 18 13:19:33.063 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:33.063 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:33.063 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:33.063 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:33.132 [conn280] end connection 127.0.0.5:46146 (6 connections now open) m31001| Fri Jan 18 13:19:33.145 [initandlisten] connection accepted from 127.0.0.5:52756 #281 (7 connections now open) m31001| Fri Jan 18 13:19:33.196 [conn278] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|4300 } } cursorid:1040280613453623 ntoreturn:0 keyUpdates:0 numYields: 9 locks(micros) r:261856 nreturned:244395 reslen:4154735 207ms m31003| Fri Jan 18 13:19:33.344 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:33.344 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:33.344 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:33.344 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31004| Fri Jan 18 13:19:33.358 [conn30] end connection 127.0.0.2:58978 (4 connections now open) m31004| Fri Jan 18 13:19:33.358 [initandlisten] connection accepted from 127.0.0.2:46697 #33 (5 connections now open) m31001| Fri Jan 18 13:19:33.376 [initandlisten] connection accepted from 127.0.0.5:58068 #282 (8 connections now open) m31001| Fri Jan 18 13:19:33.395 [conn281] end connection 127.0.0.5:52756 (7 connections now open) m31001| Fri Jan 18 13:19:33.565 [conn282] end connection 127.0.0.5:58068 (6 connections now open) m31003| Fri Jan 18 13:19:33.567 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:33.567 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:33.567 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:33.567 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:33.567 [initandlisten] connection accepted from 127.0.0.5:32875 #283 (7 connections now open) m31001| Fri Jan 18 13:19:33.638 [conn283] end connection 127.0.0.5:32875 (6 connections now open) m31003| Fri Jan 18 13:19:33.640 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:33.640 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:33.640 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:33.640 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:33.712 [initandlisten] connection accepted from 127.0.0.5:34971 #284 (7 connections now open) m31001| Fri Jan 18 13:19:33.836 [conn284] end connection 127.0.0.5:34971 (6 connections now open) m31003| Fri Jan 18 13:19:33.838 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:33.838 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:33.838 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:33.838 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:33.905 [initandlisten] connection accepted from 127.0.0.5:35421 #285 (7 connections now open) m31001| Fri Jan 18 13:19:34.029 [conn285] end connection 127.0.0.5:35421 (6 connections now open) m31003| Fri Jan 18 13:19:34.032 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:34.032 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:34.032 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:34.032 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:34.115 [initandlisten] connection accepted from 127.0.0.5:47760 #286 (7 connections now open) m31003| Fri Jan 18 13:19:34.271 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:34.271 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:34.271 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:34.271 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:34.284 [initandlisten] connection accepted from 127.0.0.5:48435 #287 (8 connections now open) m31001| Fri Jan 18 13:19:34.284 [conn286] end connection 127.0.0.5:47760 (7 connections now open) m31003| Fri Jan 18 13:19:34.315 [conn28] end connection 127.0.0.6:60806 (3 connections now open) m31003| Fri Jan 18 13:19:34.315 [initandlisten] connection accepted from 127.0.0.6:48938 #31 (5 connections now open) m31004| Fri Jan 18 13:19:34.338 [conn31] end connection 127.0.0.3:50352 (4 connections now open) m31004| Fri Jan 18 13:19:34.338 [initandlisten] connection accepted from 127.0.0.3:50907 #34 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31003| Fri Jan 18 13:19:34.416 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:34.416 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:34.416 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:34.416 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 247, "optime" : { "t" : 1358533174000, "i" : 4375 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 213, "optime" : { "t" : 1358533130000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:34Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 213, "optime" : { "t" : 1358533133000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 213, "optime" : { "t" : 1358533173000, "i" : 3410 }, "optimeDate" : ISODate("2013-01-18T18:19:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 212, "optime" : { "t" : 1358533174000, "i" : 3266 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 247, "optime" : { "t" : 1358533133000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:53Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533133000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533011000, "i" : 5100 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533174000, "i" : 3259 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 212, "optime" : { "t" : 1358533172000, "i" : 7214 }, "optimeDate" : ISODate("2013-01-18T18:19:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533130000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:34Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 246, "optime" : { "t" : 1358533134000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:54Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533011000, "i" : 4400 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533172000, "i" : 7212 }, "optimeDate" : ISODate("2013-01-18T18:19:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:34Z"), "myState" : 2, "syncingTo" : "127.0.0.3:31001", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 213, "optime" : { "t" : 1358533133000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:34Z"), "pingMs" : 27 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 213, "optime" : { "t" : 1358533130000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:18:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:33Z"), "pingMs" : 36 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 245, "optime" : { "t" : 1358533011000, "i" : 5200 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "errmsg" : "syncing to: 127.0.0.3:31001", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 213, "optime" : { "t" : 1358533173000, "i" : 1373 }, "optimeDate" : ISODate("2013-01-18T18:19:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 212, "optime" : { "t" : 1358533174000, "i" : 3266 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533133000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:34Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533133000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 212, "optime" : { "t" : 1358533011000, "i" : 5100 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 245, "optime" : { "t" : 1358533174000, "i" : 4375 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.3:31001" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:19:34.492 [initandlisten] connection accepted from 127.0.0.5:52970 #288 (8 connections now open) m31001| Fri Jan 18 13:19:34.492 [conn287] end connection 127.0.0.5:48435 (7 connections now open) m31003| Fri Jan 18 13:19:34.657 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:34.657 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:34.657 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:34.657 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:34.738 [conn288] end connection 127.0.0.5:52970 (6 connections now open) m31001| Fri Jan 18 13:19:34.835 [initandlisten] connection accepted from 127.0.0.5:33740 #289 (7 connections now open) m31003| Fri Jan 18 13:19:34.918 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:34.918 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:34.918 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:34.918 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31004| Fri Jan 18 13:19:35.007 [conn32] end connection 127.0.0.4:45719 (4 connections now open) m31004| Fri Jan 18 13:19:35.007 [initandlisten] connection accepted from 127.0.0.4:47061 #35 (5 connections now open) m31001| Fri Jan 18 13:19:35.008 [initandlisten] connection accepted from 127.0.0.5:56859 #290 (8 connections now open) m31001| Fri Jan 18 13:19:35.008 [conn289] end connection 127.0.0.5:33740 (7 connections now open) m31001| Fri Jan 18 13:19:35.351 [conn290] end connection 127.0.0.5:56859 (6 connections now open) m31003| Fri Jan 18 13:19:35.354 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:35.354 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:35.354 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:35.354 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:35.354 [initandlisten] connection accepted from 127.0.0.5:43512 #291 (7 connections now open) m31002| Fri Jan 18 13:19:35.358 [conn96] end connection 127.0.0.2:47652 (4 connections now open) m31002| Fri Jan 18 13:19:35.358 [initandlisten] connection accepted from 127.0.0.2:48411 #141 (5 connections now open) m31001| Fri Jan 18 13:19:35.544 [conn291] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|5500 } } cursorid:1051762020101791 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1003 locks(micros) r:169804 nreturned:101 reslen:10423 117ms m31003| Fri Jan 18 13:19:35.546 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:35.546 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:35.546 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:35.546 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:35.631 [conn291] end connection 127.0.0.5:43512 (6 connections now open) m31001| Fri Jan 18 13:19:35.631 [initandlisten] connection accepted from 127.0.0.5:47619 #292 (8 connections now open) m31003| Fri Jan 18 13:19:35.747 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:35.747 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:35.747 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:35.747 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:35.834 [conn292] end connection 127.0.0.5:47619 (6 connections now open) m31001| Fri Jan 18 13:19:35.909 [initandlisten] connection accepted from 127.0.0.5:49554 #293 (7 connections now open) m31003| Fri Jan 18 13:19:35.997 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:35.997 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:35.997 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:35.997 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:36.071 [initandlisten] connection accepted from 127.0.0.5:47470 #294 (8 connections now open) m31001| Fri Jan 18 13:19:36.071 [conn293] end connection 127.0.0.5:49554 (7 connections now open) m31001| Fri Jan 18 13:19:36.146 [conn294] end connection 127.0.0.5:47470 (6 connections now open) m31003| Fri Jan 18 13:19:36.150 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:36.150 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:36.150 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:36.150 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:36.221 [initandlisten] connection accepted from 127.0.0.5:44055 #295 (7 connections now open) m31001| Fri Jan 18 13:19:36.315 [conn193] end connection 127.0.0.6:39469 (6 connections now open) m31001| Fri Jan 18 13:19:36.316 [initandlisten] connection accepted from 127.0.0.6:42716 #296 (7 connections now open) m31001| Fri Jan 18 13:19:36.334 [conn295] end connection 127.0.0.5:44055 (6 connections now open) m31003| Fri Jan 18 13:19:36.337 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:36.337 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:36.337 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:36.337 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31002| Fri Jan 18 13:19:36.338 [conn99] end connection 127.0.0.3:40898 (4 connections now open) m31002| Fri Jan 18 13:19:36.339 [initandlisten] connection accepted from 127.0.0.3:41624 #142 (5 connections now open) m31001| Fri Jan 18 13:19:36.426 [initandlisten] connection accepted from 127.0.0.5:35516 #297 (7 connections now open) m31001| Fri Jan 18 13:19:36.565 [conn297] end connection 127.0.0.5:35516 (6 connections now open) m31003| Fri Jan 18 13:19:36.566 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:36.566 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:36.566 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:36.566 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:36.670 [initandlisten] connection accepted from 127.0.0.5:56932 #298 (7 connections now open) m31003| Fri Jan 18 13:19:36.855 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:36.855 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:36.855 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:36.855 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:36.913 [initandlisten] connection accepted from 127.0.0.5:36771 #299 (8 connections now open) m31001| Fri Jan 18 13:19:36.913 [conn298] end connection 127.0.0.5:56932 (7 connections now open) m31001| Fri Jan 18 13:19:37.007 [conn195] end connection 127.0.0.4:45186 (6 connections now open) m31001| Fri Jan 18 13:19:37.007 [initandlisten] connection accepted from 127.0.0.4:37685 #300 (7 connections now open) m31001| Fri Jan 18 13:19:37.046 [conn299] end connection 127.0.0.5:36771 (6 connections now open) m31003| Fri Jan 18 13:19:37.050 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:37.050 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:37.050 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:37.050 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:37.121 [initandlisten] connection accepted from 127.0.0.5:44546 #301 (7 connections now open) m31003| Fri Jan 18 13:19:37.279 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:37.279 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:37.279 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:37.279 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:37.373 [conn301] end connection 127.0.0.5:44546 (6 connections now open) m31001| Fri Jan 18 13:19:37.430 [initandlisten] connection accepted from 127.0.0.5:36330 #302 (7 connections now open) m31003| Fri Jan 18 13:19:37.510 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:37.510 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:37.510 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:37.510 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:37.578 [initandlisten] connection accepted from 127.0.0.5:46937 #303 (8 connections now open) m31001| Fri Jan 18 13:19:37.578 [conn302] end connection 127.0.0.5:36330 (7 connections now open) m31003| Fri Jan 18 13:19:37.733 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:37.733 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:37.733 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:37.733 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:37.827 [initandlisten] connection accepted from 127.0.0.5:60007 #304 (8 connections now open) m31001| Fri Jan 18 13:19:37.827 [conn303] end connection 127.0.0.5:46937 (6 connections now open) m31001| Fri Jan 18 13:19:37.998 [conn304] end connection 127.0.0.5:60007 (6 connections now open) m31003| Fri Jan 18 13:19:38.000 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:38.000 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:38.000 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:38.000 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:38.056 [initandlisten] connection accepted from 127.0.0.5:43610 #305 (7 connections now open) m31003| Fri Jan 18 13:19:38.122 [rsMgr] replSet I don't see a primary and I can't elect myself m31001| Fri Jan 18 13:19:38.169 [conn305] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|6700 } } cursorid:1062932852681704 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1030 locks(micros) r:220312 nreturned:101 reslen:10423 112ms m31003| Fri Jan 18 13:19:38.171 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:38.171 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:38.171 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:38.171 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:38.227 [conn305] end connection 127.0.0.5:43610 (6 connections now open) m31001| Fri Jan 18 13:19:38.227 [conn200] end connection 127.0.0.5:58358 (6 connections now open) m31001| Fri Jan 18 13:19:38.273 [initandlisten] connection accepted from 127.0.0.5:50416 #306 (6 connections now open) m31001| Fri Jan 18 13:19:38.282 [initandlisten] connection accepted from 127.0.0.5:52383 #307 (7 connections now open) m31000| Fri Jan 18 13:19:38.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:19:38.359 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:38.359 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:38.359 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:38.359 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:38.412 [initandlisten] connection accepted from 127.0.0.5:45741 #308 (8 connections now open) m31001| Fri Jan 18 13:19:38.412 [conn306] end connection 127.0.0.5:50416 (6 connections now open) m31003| Fri Jan 18 13:19:38.608 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:38.608 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:38.608 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:38.608 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:38.609 [conn308] end connection 127.0.0.5:45741 (6 connections now open) m31001| Fri Jan 18 13:19:38.609 [initandlisten] connection accepted from 127.0.0.5:37985 #309 (8 connections now open) m31003| Fri Jan 18 13:19:38.818 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:38.818 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:38.818 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:38.818 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:38.903 [initandlisten] connection accepted from 127.0.0.5:44173 #310 (8 connections now open) m31001| Fri Jan 18 13:19:38.903 [conn309] end connection 127.0.0.5:37985 (7 connections now open) m31001| Fri Jan 18 13:19:39.037 [conn310] end connection 127.0.0.5:44173 (6 connections now open) m31003| Fri Jan 18 13:19:39.039 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:39.039 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:39.039 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:39.039 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:39.135 [initandlisten] connection accepted from 127.0.0.5:35534 #311 (7 connections now open) m31003| Fri Jan 18 13:19:39.360 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:39.360 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:39.360 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:39.360 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:39.452 [conn311] end connection 127.0.0.5:35534 (6 connections now open) m31001| Fri Jan 18 13:19:39.472 [initandlisten] connection accepted from 127.0.0.5:32898 #312 (7 connections now open) m31003| Fri Jan 18 13:19:39.611 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:39.611 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:39.611 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:39.611 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:39.618 [conn312] end connection 127.0.0.5:32898 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 252, "optime" : { "t" : 1358533179000, "i" : 5560 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358533136000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358533136000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358533179000, "i" : 3014 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 217, "optime" : { "t" : 1358533178000, "i" : 2864 }, "optimeDate" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 252, "optime" : { "t" : 1358533139000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:59Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358533136000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:39Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358533011000, "i" : 6800 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358533178000, "i" : 2750 }, "optimeDate" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 217, "optime" : { "t" : 1358533179000, "i" : 97 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358533136000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 251, "optime" : { "t" : 1358533136000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358533011000, "i" : 7100 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358533178000, "i" : 9054 }, "optimeDate" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:39Z"), "myState" : 2, "syncingTo" : "127.0.0.3:31001", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358533136000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:38Z"), "pingMs" : 24 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358533136000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:39Z"), "pingMs" : 25 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 250, "optime" : { "t" : 1358533011000, "i" : 7400 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "errmsg" : "syncing to: 127.0.0.3:31001", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358533178000, "i" : 1095 }, "optimeDate" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 217, "optime" : { "t" : 1358533178000, "i" : 2673 }, "optimeDate" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358533136000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:38Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358533136000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:18:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:39Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358533011000, "i" : 6800 }, "optimeDate" : ISODate("2013-01-18T18:16:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.3:31001" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 250, "optime" : { "t" : 1358533179000, "i" : 5560 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.3:31001" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:19:39.781 [initandlisten] connection accepted from 127.0.0.5:57224 #313 (7 connections now open) m31003| Fri Jan 18 13:19:39.942 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:39.942 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:39.942 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:39.942 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:40.008 [conn313] end connection 127.0.0.5:57224 (6 connections now open) m31001| Fri Jan 18 13:19:40.008 [initandlisten] connection accepted from 127.0.0.5:33669 #314 (8 connections now open) m31001| Fri Jan 18 13:19:40.130 [conn314] end connection 127.0.0.5:33669 (6 connections now open) m31003| Fri Jan 18 13:19:40.133 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:40.133 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:40.133 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:40.133 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:40.145 [initandlisten] connection accepted from 127.0.0.5:45083 #315 (7 connections now open) m31003| Fri Jan 18 13:19:40.302 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:40.302 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:40.302 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:40.302 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:40.370 [conn315] end connection 127.0.0.5:45083 (6 connections now open) m31001| Fri Jan 18 13:19:40.380 [initandlisten] connection accepted from 127.0.0.5:37801 #316 (7 connections now open) m31003| Fri Jan 18 13:19:40.496 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:40.496 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:40.496 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:19:40.496 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31001| Fri Jan 18 13:19:40.525 [conn316] end connection 127.0.0.5:37801 (6 connections now open) m31002| Fri Jan 18 13:19:40.543 [initandlisten] connection accepted from 127.0.0.5:47447 #143 (6 connections now open) m31004| Fri Jan 18 13:19:40.623 [conn29] end connection 127.0.0.5:51869 (4 connections now open) m31003| Fri Jan 18 13:19:40.707 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 m31002| Fri Jan 18 13:19:40.767 [conn143] end connection 127.0.0.5:47447 (5 connections now open) m31002| Fri Jan 18 13:19:40.767 [initandlisten] connection accepted from 127.0.0.5:41030 #144 (7 connections now open) m31003| Fri Jan 18 13:19:40.768 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:40.768 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:40.768 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:40.768 [initandlisten] connection accepted from 127.0.0.5:34333 #145 (7 connections now open) m31001| Fri Jan 18 13:19:40.798 [conn278] end connection 127.0.0.5:58538 (5 connections now open) m31003| Fri Jan 18 13:19:40.862 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:40.862 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:40.862 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:40.956 [initandlisten] connection accepted from 127.0.0.5:43441 #146 (8 connections now open) m31002| Fri Jan 18 13:19:40.956 [conn145] end connection 127.0.0.5:34333 (6 connections now open) m31004| Fri Jan 18 13:19:41.123 [initandlisten] connection accepted from 127.0.0.5:55049 #36 (5 connections now open) m31002| Fri Jan 18 13:19:41.234 [conn146] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|8000 } } cursorid:1073510414061197 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1051 locks(micros) r:181558 nreturned:101 reslen:10423 240ms m31003| Fri Jan 18 13:19:41.237 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:41.237 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:41.237 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:41.258 [conn146] end connection 127.0.0.5:43441 (6 connections now open) m31002| Fri Jan 18 13:19:41.323 [initandlisten] connection accepted from 127.0.0.5:58352 #147 (7 connections now open) m31002| Fri Jan 18 13:19:41.563 [conn144] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|7900 } } cursorid:1071895068737491 ntoreturn:0 keyUpdates:0 numYields: 1381 locks(micros) r:502878 nreturned:246722 reslen:4194294 608ms m31002| Fri Jan 18 13:19:41.622 [conn147] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533011000|8100 } } cursorid:1075155763469314 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1076 locks(micros) r:180680 nreturned:101 reslen:10423 117ms m31003| Fri Jan 18 13:19:41.626 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:41.626 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:41.626 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:41.677 [initandlisten] connection accepted from 127.0.0.5:44303 #148 (8 connections now open) m31002| Fri Jan 18 13:19:41.689 [conn147] end connection 127.0.0.5:58352 (7 connections now open) m31002| Fri Jan 18 13:19:41.846 [conn148] end connection 127.0.0.5:44303 (6 connections now open) m31003| Fri Jan 18 13:19:41.847 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:41.847 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:41.847 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:41.897 [initandlisten] connection accepted from 127.0.0.5:44256 #149 (7 connections now open) m31003| Fri Jan 18 13:19:42.040 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:42.040 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:42.040 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:42.075 [conn149] end connection 127.0.0.5:44256 (6 connections now open) m31002| Fri Jan 18 13:19:42.075 [initandlisten] connection accepted from 127.0.0.5:34775 #150 (8 connections now open) m31003| Fri Jan 18 13:19:42.165 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:42.165 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:42.165 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:42.258 [initandlisten] connection accepted from 127.0.0.5:43609 #151 (8 connections now open) m31002| Fri Jan 18 13:19:42.258 [conn150] end connection 127.0.0.5:34775 (7 connections now open) m31002| Fri Jan 18 13:19:42.346 [conn151] end connection 127.0.0.5:43609 (6 connections now open) m31003| Fri Jan 18 13:19:42.350 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:42.350 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:42.350 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:42.444 [initandlisten] connection accepted from 127.0.0.5:45276 #152 (7 connections now open) m31003| Fri Jan 18 13:19:42.527 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:42.527 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:42.527 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:42.535 [conn152] end connection 127.0.0.5:45276 (6 connections now open) m31002| Fri Jan 18 13:19:42.535 [initandlisten] connection accepted from 127.0.0.5:57517 #153 (8 connections now open) m31003| Fri Jan 18 13:19:42.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:19:42.645 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:42.645 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:42.645 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:42.693 [initandlisten] connection accepted from 127.0.0.5:53512 #154 (8 connections now open) m31002| Fri Jan 18 13:19:42.693 [conn153] end connection 127.0.0.5:57517 (7 connections now open) m31003| Fri Jan 18 13:19:42.811 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:42.811 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:42.811 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:42.850 [conn154] end connection 127.0.0.5:53512 (6 connections now open) m31002| Fri Jan 18 13:19:42.945 [initandlisten] connection accepted from 127.0.0.5:49485 #155 (7 connections now open) m31003| Fri Jan 18 13:19:43.087 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:43.088 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:43.088 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:43.136 [conn155] end connection 127.0.0.5:49485 (6 connections now open) m31002| Fri Jan 18 13:19:43.152 [initandlisten] connection accepted from 127.0.0.5:46441 #156 (7 connections now open) m31003| Fri Jan 18 13:19:43.276 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:43.276 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:43.276 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:43.339 [conn156] end connection 127.0.0.5:46441 (6 connections now open) m31002| Fri Jan 18 13:19:43.339 [initandlisten] connection accepted from 127.0.0.5:44507 #157 (8 connections now open) m31003| Fri Jan 18 13:19:43.521 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:43.521 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:43.521 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:43.602 [conn157] end connection 127.0.0.5:44507 (6 connections now open) m31002| Fri Jan 18 13:19:43.699 [initandlisten] connection accepted from 127.0.0.5:51441 #158 (7 connections now open) m31002| Fri Jan 18 13:19:43.859 [conn158] end connection 127.0.0.5:51441 (6 connections now open) m31003| Fri Jan 18 13:19:43.861 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:43.861 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:43.861 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:43.956 [initandlisten] connection accepted from 127.0.0.5:36893 #159 (7 connections now open) m31003| Fri Jan 18 13:19:44.076 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:44.076 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:44.076 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:44.141 [conn159] end connection 127.0.0.5:36893 (6 connections now open) m31002| Fri Jan 18 13:19:44.141 [initandlisten] connection accepted from 127.0.0.5:35914 #160 (8 connections now open) m31002| Fri Jan 18 13:19:44.302 [conn160] end connection 127.0.0.5:35914 (6 connections now open) m31003| Fri Jan 18 13:19:44.304 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:44.304 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:44.304 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:44.355 [initandlisten] connection accepted from 127.0.0.5:36494 #161 (7 connections now open) m31003| Fri Jan 18 13:19:44.438 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:44.438 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:44.438 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:44.491 [initandlisten] connection accepted from 127.0.0.5:56934 #162 (8 connections now open) m31002| Fri Jan 18 13:19:44.491 [conn161] end connection 127.0.0.5:36494 (7 connections now open) m31003| Fri Jan 18 13:19:44.604 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:44.604 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:44.604 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:44.629 [initandlisten] connection accepted from 127.0.0.5:42489 #163 (8 connections now open) m31002| Fri Jan 18 13:19:44.629 [conn162] end connection 127.0.0.5:56934 (7 connections now open) m31002| Fri Jan 18 13:19:44.789 [conn163] end connection 127.0.0.5:42489 (6 connections now open) m31003| Fri Jan 18 13:19:44.792 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:44.792 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:44.792 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:44.877 [initandlisten] connection accepted from 127.0.0.5:36663 #164 (7 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 258, "optime" : { "t" : 1358533185000, "i" : 107 }, "optimeDate" : ISODate("2013-01-18T18:19:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 224, "optime" : { "t" : 1358533141000, "i" : 125 }, "optimeDate" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 224, "optime" : { "t" : 1358533141000, "i" : 125 }, "optimeDate" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:45Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 224, "optime" : { "t" : 1358533183000, "i" : 3113 }, "optimeDate" : ISODate("2013-01-18T18:19:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:44Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 223, "optime" : { "t" : 1358533184000, "i" : 2989 }, "optimeDate" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 258, "optime" : { "t" : 1358533144000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:04Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533141000, "i" : 125 }, "optimeDate" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:45Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533012000, "i" : 804 }, "optimeDate" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:44Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533184000, "i" : 2969 }, "optimeDate" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 223, "optime" : { "t" : 1358533185000, "i" : 34 }, "optimeDate" : ISODate("2013-01-18T18:19:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533144000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 257, "optime" : { "t" : 1358533144000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:04Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533012000, "i" : 1104 }, "optimeDate" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533185000, "i" : 28 }, "optimeDate" : ISODate("2013-01-18T18:19:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 m31002| Fri Jan 18 13:19:45.024 [conn164] end connection 127.0.0.5:36663 (6 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:45Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 224, "optime" : { "t" : 1358533144000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 38 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 224, "optime" : { "t" : 1358533141000, "i" : 125 }, "optimeDate" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:45Z"), "pingMs" : 25 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358533012000, "i" : 1104 }, "optimeDate" : ISODate("2013-01-18T18:16:52Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 224, "optime" : { "t" : 1358533183000, "i" : 4672 }, "optimeDate" : ISODate("2013-01-18T18:19:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:44Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 223, "optime" : { "t" : 1358533184000, "i" : 2820 }, "optimeDate" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533144000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533141000, "i" : 125 }, "optimeDate" : ISODate("2013-01-18T18:19:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:45Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358533012000, "i" : 804 }, "optimeDate" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358533185000, "i" : 107 }, "optimeDate" : ISODate("2013-01-18T18:19:45Z"), "self" : true } ], "ok" : 1 } m31003| Fri Jan 18 13:19:45.026 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:45.026 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:45.026 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31002| Fri Jan 18 13:19:45.078 [initandlisten] connection accepted from 127.0.0.5:41601 #165 (7 connections now open) m31002| Fri Jan 18 13:19:45.237 [conn165] end connection 127.0.0.5:41601 (6 connections now open) m31003| Fri Jan 18 13:19:45.239 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:45.239 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:45.239 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:45.328 [initandlisten] connection accepted from 127.0.0.5:60503 #166 (7 connections now open) m31003| Fri Jan 18 13:19:45.469 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:45.469 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:45.469 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:45.554 [conn166] end connection 127.0.0.5:60503 (6 connections now open) m31002| Fri Jan 18 13:19:45.554 [initandlisten] connection accepted from 127.0.0.5:34360 #167 (8 connections now open) m31003| Fri Jan 18 13:19:45.707 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:45.707 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:45.707 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:45.792 [conn167] end connection 127.0.0.5:34360 (6 connections now open) m31002| Fri Jan 18 13:19:45.813 [initandlisten] connection accepted from 127.0.0.5:46144 #168 (7 connections now open) m31003| Fri Jan 18 13:19:45.976 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:45.976 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:45.976 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:46.072 [conn168] end connection 127.0.0.5:46144 (6 connections now open) m31002| Fri Jan 18 13:19:46.072 [initandlisten] connection accepted from 127.0.0.5:34035 #169 (7 connections now open) m31002| Fri Jan 18 13:19:46.248 [conn169] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533012000|1604 } } cursorid:1094838735392590 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1115 locks(micros) r:182968 nreturned:101 reslen:10423 158ms m31003| Fri Jan 18 13:19:46.251 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:46.251 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:46.251 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:46.327 [conn169] end connection 127.0.0.5:34035 (6 connections now open) m31002| Fri Jan 18 13:19:46.327 [initandlisten] connection accepted from 127.0.0.5:41793 #170 (8 connections now open) m31002| Fri Jan 18 13:19:46.435 [conn170] end connection 127.0.0.5:41793 (6 connections now open) m31003| Fri Jan 18 13:19:46.614 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:46.614 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:46.614 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:46.643 [initandlisten] connection accepted from 127.0.0.5:38174 #171 (7 connections now open) m31003| Fri Jan 18 13:19:46.879 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:46.879 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:46.879 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:46.944 [conn171] end connection 127.0.0.5:38174 (6 connections now open) m31002| Fri Jan 18 13:19:46.944 [initandlisten] connection accepted from 127.0.0.5:44418 #172 (8 connections now open) m31003| Fri Jan 18 13:19:47.028 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:47.028 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:47.028 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:47.045 [conn172] end connection 127.0.0.5:44418 (6 connections now open) m31002| Fri Jan 18 13:19:47.114 [initandlisten] connection accepted from 127.0.0.5:51887 #173 (7 connections now open) m31003| Fri Jan 18 13:19:47.202 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:47.202 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:47.202 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:47.271 [conn173] end connection 127.0.0.5:51887 (6 connections now open) m31002| Fri Jan 18 13:19:47.271 [initandlisten] connection accepted from 127.0.0.5:49006 #174 (8 connections now open) m31003| Fri Jan 18 13:19:47.425 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:47.425 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:47.425 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:47.443 [conn174] end connection 127.0.0.5:49006 (6 connections now open) m31002| Fri Jan 18 13:19:47.482 [initandlisten] connection accepted from 127.0.0.5:58252 #175 (7 connections now open) m31003| Fri Jan 18 13:19:47.596 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:47.596 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:47.596 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:47.665 [conn175] end connection 127.0.0.5:58252 (6 connections now open) m31002| Fri Jan 18 13:19:47.665 [initandlisten] connection accepted from 127.0.0.5:46065 #176 (8 connections now open) m31003| Fri Jan 18 13:19:47.940 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:47.940 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:47.940 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:47.987 [conn176] end connection 127.0.0.5:46065 (6 connections now open) m31002| Fri Jan 18 13:19:47.987 [initandlisten] connection accepted from 127.0.0.5:59657 #177 (8 connections now open) m31003| Fri Jan 18 13:19:48.263 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:48.263 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:48.263 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:48.305 [conn177] end connection 127.0.0.5:59657 (6 connections now open) m31002| Fri Jan 18 13:19:48.305 [initandlisten] connection accepted from 127.0.0.5:60748 #178 (8 connections now open) m31003| Fri Jan 18 13:19:48.502 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:48.502 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:48.502 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:48.581 [conn178] end connection 127.0.0.5:60748 (6 connections now open) m31002| Fri Jan 18 13:19:48.599 [initandlisten] connection accepted from 127.0.0.5:55821 #179 (7 connections now open) m31003| Fri Jan 18 13:19:48.791 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:48.791 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:48.791 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:48.841 [initandlisten] connection accepted from 127.0.0.5:39865 #180 (8 connections now open) m31002| Fri Jan 18 13:19:48.841 [conn179] end connection 127.0.0.5:55821 (6 connections now open) m31003| Fri Jan 18 13:19:48.946 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:48.946 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:48.946 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:49.006 [conn180] end connection 127.0.0.5:39865 (6 connections now open) m31002| Fri Jan 18 13:19:49.010 [initandlisten] connection accepted from 127.0.0.5:39740 #181 (7 connections now open) m31003| Fri Jan 18 13:19:49.178 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:49.178 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:49.178 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:49.224 [conn181] end connection 127.0.0.5:39740 (6 connections now open) m31002| Fri Jan 18 13:19:49.224 [initandlisten] connection accepted from 127.0.0.5:44046 #182 (8 connections now open) m31003| Fri Jan 18 13:19:49.405 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:49.405 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:49.405 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:49.455 [initandlisten] connection accepted from 127.0.0.5:60838 #183 (8 connections now open) m31002| Fri Jan 18 13:19:49.455 [conn182] end connection 127.0.0.5:44046 (6 connections now open) m31002| Fri Jan 18 13:19:49.602 [conn183] end connection 127.0.0.5:60838 (6 connections now open) m31003| Fri Jan 18 13:19:49.604 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:49.604 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:49.604 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:49.662 [initandlisten] connection accepted from 127.0.0.5:51375 #184 (7 connections now open) m31003| Fri Jan 18 13:19:49.773 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:49.773 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:49.773 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:49.798 [conn184] end connection 127.0.0.5:51375 (6 connections now open) m31002| Fri Jan 18 13:19:49.913 [initandlisten] connection accepted from 127.0.0.5:53347 #185 (7 connections now open) m31003| Fri Jan 18 13:19:50.011 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:50.011 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:50.011 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:50.060 [conn185] end connection 127.0.0.5:53347 (6 connections now open) m31002| Fri Jan 18 13:19:50.071 [initandlisten] connection accepted from 127.0.0.5:36246 #186 (7 connections now open) m31003| Fri Jan 18 13:19:50.169 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:50.169 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:50.169 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:50.185 [conn186] end connection 127.0.0.5:36246 (6 connections now open) m31002| Fri Jan 18 13:19:50.185 [initandlisten] connection accepted from 127.0.0.5:41626 #187 (8 connections now open) m31002| Fri Jan 18 13:19:50.317 [conn111] end connection 127.0.0.6:39608 (6 connections now open) m31002| Fri Jan 18 13:19:50.318 [initandlisten] connection accepted from 127.0.0.6:54525 #188 (7 connections now open) m31003| Fri Jan 18 13:19:50.326 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:50.326 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:50.326 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:19:50.340 [conn29] end connection 127.0.0.3:38716 (3 connections now open) m31003| Fri Jan 18 13:19:50.340 [initandlisten] connection accepted from 127.0.0.3:38516 #32 (4 connections now open) m31002| Fri Jan 18 13:19:50.344 [conn187] end connection 127.0.0.5:41626 (6 connections now open) m31002| Fri Jan 18 13:19:50.390 [initandlisten] connection accepted from 127.0.0.5:33098 #189 (7 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 263, "optime" : { "t" : 1358533190000, "i" : 3449 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 229, "optime" : { "t" : 1358533147000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:50Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 229, "optime" : { "t" : 1358533147000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 229, "optime" : { "t" : 1358533189000, "i" : 2542 }, "optimeDate" : ISODate("2013-01-18T18:19:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:50Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 228, "optime" : { "t" : 1358533190000, "i" : 2519 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 263, "optime" : { "t" : 1358533147000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533147000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533015000, "i" : 600 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533190000, "i" : 2504 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 228, "optime" : { "t" : 1358533189000, "i" : 38 }, "optimeDate" : ISODate("2013-01-18T18:19:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533147000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:50Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 262, "optime" : { "t" : 1358533147000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533012000, "i" : 2804 }, "optimeDate" : ISODate("2013-01-18T18:16:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533189000, "i" : 31 }, "optimeDate" : ISODate("2013-01-18T18:19:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:50Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 m31003| Fri Jan 18 13:19:50.557 [conn1] command admin.$cmd command: { replSetGetStatus: true } ntoreturn:1 keyUpdates:0 reslen:1149 137ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:50Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 229, "optime" : { "t" : 1358533147000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:50Z"), "pingMs" : 32 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 229, "optime" : { "t" : 1358533147000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:49Z"), "pingMs" : 23 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 261, "optime" : { "t" : 1358533015000, "i" : 600 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 229, "optime" : { "t" : 1358533188000, "i" : 4428 }, "optimeDate" : ISODate("2013-01-18T18:19:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:50.559 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:50.559 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:50.559 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 228, "optime" : { "t" : 1358533190000, "i" : 2519 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533147000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:50Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533147000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 228, "optime" : { "t" : 1358533015000, "i" : 500 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 261, "optime" : { "t" : 1358533190000, "i" : 3449 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31002| Fri Jan 18 13:19:50.608 [conn189] end connection 127.0.0.5:33098 (6 connections now open) m31002| Fri Jan 18 13:19:50.667 [initandlisten] connection accepted from 127.0.0.5:50665 #190 (7 connections now open) m31003| Fri Jan 18 13:19:50.832 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:50.832 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:50.832 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:50.924 [conn190] end connection 127.0.0.5:50665 (6 connections now open) m31002| Fri Jan 18 13:19:50.925 [initandlisten] connection accepted from 127.0.0.5:36930 #191 (7 connections now open) m31003| Fri Jan 18 13:19:51.009 [conn30] end connection 127.0.0.4:48346 (3 connections now open) m31003| Fri Jan 18 13:19:51.009 [initandlisten] connection accepted from 127.0.0.4:59941 #33 (4 connections now open) m31000| Fri Jan 18 13:19:51.046 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:146493 nreturned:160000 reslen:2720020 146ms m31000| Fri Jan 18 13:19:51.064 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:138637 nreturned:160000 reslen:2720020 138ms m31002| Fri Jan 18 13:19:51.203 [conn191] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|800 } } cursorid:1115902232762943 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1188 locks(micros) r:199846 nreturned:101 reslen:10423 101ms m31003| Fri Jan 18 13:19:51.205 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:51.205 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:51.205 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:51.242 [conn191] end connection 127.0.0.5:36930 (6 connections now open) m31002| Fri Jan 18 13:19:51.294 [initandlisten] connection accepted from 127.0.0.5:46296 #192 (7 connections now open) m31001| Fri Jan 18 13:19:51.360 [conn256] end connection 127.0.0.2:38277 (4 connections now open) m31001| Fri Jan 18 13:19:51.361 [initandlisten] connection accepted from 127.0.0.2:49978 #317 (5 connections now open) m31002| Fri Jan 18 13:19:51.549 [conn192] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|900 } } cursorid:1117388952630840 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1207 locks(micros) r:197171 nreturned:101 reslen:10423 120ms m31002| Fri Jan 18 13:19:51.556 [conn192] end connection 127.0.0.5:46296 (6 connections now open) m31003| Fri Jan 18 13:19:51.592 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:51.592 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:51.592 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:51.715 [initandlisten] connection accepted from 127.0.0.5:55947 #193 (7 connections now open) m31002| Fri Jan 18 13:19:51.982 [conn193] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|1000 } } cursorid:1119218148079821 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1226 locks(micros) r:214571 nreturned:101 reslen:10423 108ms m31003| Fri Jan 18 13:19:51.987 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:51.987 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:51.987 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:52.069 [conn193] end connection 127.0.0.5:55947 (6 connections now open) m31002| Fri Jan 18 13:19:52.133 [initandlisten] connection accepted from 127.0.0.5:34927 #194 (7 connections now open) m31003| Fri Jan 18 13:19:52.303 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:52.304 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:52.304 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31000| Fri Jan 18 13:19:52.318 [conn51] end connection 127.0.0.6:36505 (11 connections now open) m31000| Fri Jan 18 13:19:52.318 [initandlisten] connection accepted from 127.0.0.6:59757 #54 (12 connections now open) m31000| Fri Jan 18 13:19:52.341 [conn52] end connection 127.0.0.3:49130 (11 connections now open) m31000| Fri Jan 18 13:19:52.341 [initandlisten] connection accepted from 127.0.0.3:41117 #55 (12 connections now open) m31002| Fri Jan 18 13:19:52.354 [conn194] end connection 127.0.0.5:34927 (6 connections now open) m31002| Fri Jan 18 13:19:52.354 [initandlisten] connection accepted from 127.0.0.5:39688 #195 (8 connections now open) m31002| Fri Jan 18 13:19:52.457 [conn195] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|1200 } } cursorid:1121244467352016 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1225 locks(micros) r:201666 nreturned:101 reslen:10423 102ms m31002| Fri Jan 18 13:19:52.457 [conn195] end connection 127.0.0.5:39688 (6 connections now open) m31003| Fri Jan 18 13:19:52.459 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:52.459 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:52.459 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:52.559 [initandlisten] connection accepted from 127.0.0.5:52627 #196 (7 connections now open) m31003| Fri Jan 18 13:19:52.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Fri Jan 18 13:19:52.674 [conn113] end connection 127.0.0.5:35530 (6 connections now open) m31002| Fri Jan 18 13:19:52.674 [initandlisten] connection accepted from 127.0.0.5:56602 #197 (7 connections now open) m31002| Fri Jan 18 13:19:52.687 [conn196] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|1300 } } cursorid:1122219244764909 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1232 locks(micros) r:203813 nreturned:101 reslen:10423 103ms m31003| Fri Jan 18 13:19:52.691 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:52.691 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:52.691 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:52.761 [initandlisten] connection accepted from 127.0.0.5:33676 #198 (8 connections now open) m31002| Fri Jan 18 13:19:52.761 [conn196] end connection 127.0.0.5:52627 (7 connections now open) m31002| Fri Jan 18 13:19:52.911 [conn198] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|1400 } } cursorid:1123143958369723 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1231 locks(micros) r:200239 nreturned:101 reslen:10423 101ms m31003| Fri Jan 18 13:19:52.916 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:52.916 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:52.916 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:52.980 [conn198] end connection 127.0.0.5:33676 (6 connections now open) m31000| Fri Jan 18 13:19:53.009 [conn53] end connection 127.0.0.4:49145 (11 connections now open) m31000| Fri Jan 18 13:19:53.009 [initandlisten] connection accepted from 127.0.0.4:55347 #56 (12 connections now open) m31002| Fri Jan 18 13:19:53.073 [initandlisten] connection accepted from 127.0.0.5:38367 #199 (7 connections now open) m31002| Fri Jan 18 13:19:53.182 [conn199] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|1500 } } cursorid:1124298014065618 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1231 locks(micros) r:213530 nreturned:101 reslen:10423 108ms m31003| Fri Jan 18 13:19:53.186 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:53.186 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:53.186 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:53.190 [conn199] end connection 127.0.0.5:38367 (6 connections now open) m31002| Fri Jan 18 13:19:53.235 [initandlisten] connection accepted from 127.0.0.5:48748 #200 (7 connections now open) m31003| Fri Jan 18 13:19:53.396 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:53.396 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:53.396 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:53.480 [initandlisten] connection accepted from 127.0.0.5:34355 #201 (8 connections now open) m31002| Fri Jan 18 13:19:53.480 [conn200] end connection 127.0.0.5:48748 (6 connections now open) m31002| Fri Jan 18 13:19:53.762 [conn201] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|1700 } } cursorid:1126755044284355 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1230 locks(micros) r:199733 nreturned:101 reslen:10423 101ms m31003| Fri Jan 18 13:19:53.764 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:53.764 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:53.764 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:53.823 [initandlisten] connection accepted from 127.0.0.5:54393 #202 (8 connections now open) m31002| Fri Jan 18 13:19:53.823 [conn201] end connection 127.0.0.5:34355 (7 connections now open) m31002| Fri Jan 18 13:19:54.050 [conn202] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|1800 } } cursorid:1128005668635577 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1230 locks(micros) r:203648 nreturned:101 reslen:10423 103ms m31003| Fri Jan 18 13:19:54.052 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:54.052 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:54.052 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:54.074 [conn202] end connection 127.0.0.5:54393 (6 connections now open) m31002| Fri Jan 18 13:19:54.143 [initandlisten] connection accepted from 127.0.0.5:45104 #203 (7 connections now open) m31003| Fri Jan 18 13:19:54.403 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:54.403 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:54.403 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:54.478 [initandlisten] connection accepted from 127.0.0.5:56090 #204 (8 connections now open) m31002| Fri Jan 18 13:19:54.478 [conn203] end connection 127.0.0.5:45104 (6 connections now open) m31002| Fri Jan 18 13:19:54.688 [conn204] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|2000 } } cursorid:1130706477311228 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1229 locks(micros) r:226720 nreturned:101 reslen:10423 114ms m31003| Fri Jan 18 13:19:54.690 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:54.690 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:54.690 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:54.752 [initandlisten] connection accepted from 127.0.0.5:38394 #205 (8 connections now open) m31002| Fri Jan 18 13:19:54.752 [conn204] end connection 127.0.0.5:56090 (6 connections now open) m31002| Fri Jan 18 13:19:54.950 [conn205] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|2100 } } cursorid:1131832533365585 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1228 locks(micros) r:204080 nreturned:101 reslen:10423 103ms m31003| Fri Jan 18 13:19:54.953 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:54.953 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:54.953 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:54.965 [conn205] end connection 127.0.0.5:38394 (6 connections now open) m31002| Fri Jan 18 13:19:55.020 [initandlisten] connection accepted from 127.0.0.5:42435 #206 (7 connections now open) m31003| Fri Jan 18 13:19:55.189 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:55.189 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:55.189 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:55.221 [conn206] end connection 127.0.0.5:42435 (6 connections now open) m31002| Fri Jan 18 13:19:55.268 [initandlisten] connection accepted from 127.0.0.5:58557 #207 (7 connections now open) m31002| Fri Jan 18 13:19:55.390 [conn207] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|2300 } } cursorid:1133725866213735 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1228 locks(micros) r:200683 nreturned:101 reslen:10423 101ms m31002| Fri Jan 18 13:19:55.391 [conn207] end connection 127.0.0.5:58557 (6 connections now open) m31003| Fri Jan 18 13:19:55.392 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:55.392 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:55.392 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:55.471 [initandlisten] connection accepted from 127.0.0.5:47348 #208 (7 connections now open) m31002| Fri Jan 18 13:19:55.610 [conn208] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|2400 } } cursorid:1134632339017371 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1227 locks(micros) r:208101 nreturned:101 reslen:10423 106ms m31003| Fri Jan 18 13:19:55.612 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:55.612 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:55.612 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:55.685 [initandlisten] connection accepted from 127.0.0.5:48647 #209 (8 connections now open) m31002| Fri Jan 18 13:19:55.685 [conn208] end connection 127.0.0.5:47348 (7 connections now open) STATUS FROM : 127.0.0.2:31000 m31002| Fri Jan 18 13:19:55.875 [conn209] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|2500 } } cursorid:1135756987625215 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1240 locks(micros) r:226866 nreturned:101 reslen:10423 114ms m31003| Fri Jan 18 13:19:55.878 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:55.878 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:55.878 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:55.884 [conn209] end connection 127.0.0.5:48647 (6 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 268, "optime" : { "t" : 1358533195000, "i" : 7815 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358533152000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358533152000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358533195000, "i" : 3376 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 233, "optime" : { "t" : 1358533194000, "i" : 3035 }, "optimeDate" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 268, "optime" : { "t" : 1358533155000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:15Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 233, "optime" : { "t" : 1358533152000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:55Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 233, "optime" : { "t" : 1358533015000, "i" : 1900 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:54Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 233, "optime" : { "t" : 1358533194000, "i" : 2849 }, "optimeDate" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 233, "optime" : { "t" : 1358533195000, "i" : 131 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 233, "optime" : { "t" : 1358533152000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 267, "optime" : { "t" : 1358533155000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:15Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 233, "optime" : { "t" : 1358533015000, "i" : 2200 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 233, "optime" : { "t" : 1358533195000, "i" : 98 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:55Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358533152000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 42 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358533152000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:55Z"), "pingMs" : 32 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 266, "optime" : { "t" : 1358533015000, "i" : 2600 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358533193000, "i" : 363 }, "optimeDate" : ISODate("2013-01-18T18:19:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:19:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 233, "optime" : { "t" : 1358533194000, "i" : 2867 }, "optimeDate" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 233, "optime" : { "t" : 1358533152000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 233, "optime" : { "t" : 1358533152000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:19:55Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 233, "optime" : { "t" : 1358533015000, "i" : 1900 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 266, "optime" : { "t" : 1358533195000, "i" : 7815 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31002| Fri Jan 18 13:19:55.972 [initandlisten] connection accepted from 127.0.0.5:37864 #210 (7 connections now open) m31002| Fri Jan 18 13:19:56.141 [conn210] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|2600 } } cursorid:1136897029228672 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1239 locks(micros) r:206545 nreturned:101 reslen:10423 104ms m31003| Fri Jan 18 13:19:56.145 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:56.145 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:56.145 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:56.236 [initandlisten] connection accepted from 127.0.0.5:48209 #211 (8 connections now open) m31002| Fri Jan 18 13:19:56.236 [conn210] end connection 127.0.0.5:37864 (6 connections now open) m31002| Fri Jan 18 13:19:56.452 [conn211] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|2700 } } cursorid:1138231464740926 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1239 locks(micros) r:203823 nreturned:101 reslen:10423 103ms m31003| Fri Jan 18 13:19:56.456 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:56.456 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:56.456 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:56.550 [conn211] end connection 127.0.0.5:48209 (6 connections now open) m31002| Fri Jan 18 13:19:56.608 [initandlisten] connection accepted from 127.0.0.5:52304 #212 (7 connections now open) m31002| Fri Jan 18 13:19:56.746 [conn212] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|2800 } } cursorid:1139468648083552 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1239 locks(micros) r:237984 nreturned:101 reslen:10423 121ms m31002| Fri Jan 18 13:19:56.746 [conn212] end connection 127.0.0.5:52304 (6 connections now open) m31003| Fri Jan 18 13:19:56.748 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:56.748 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:56.748 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:56.797 [initandlisten] connection accepted from 127.0.0.5:46300 #213 (7 connections now open) m31002| Fri Jan 18 13:19:56.971 [conn213] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|2900 } } cursorid:1140442697928098 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1253 locks(micros) r:214249 nreturned:101 reslen:10423 109ms m31003| Fri Jan 18 13:19:56.974 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:56.974 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:56.974 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:56.974 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:19:56.974 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31001| Fri Jan 18 13:19:57.008 [initandlisten] connection accepted from 127.0.0.5:53130 #318 (6 connections now open) m31002| Fri Jan 18 13:19:57.021 [conn213] end connection 127.0.0.5:46300 (6 connections now open) m31001| Fri Jan 18 13:19:57.240 [conn318] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3000 } } cursorid:1144119784617331 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1272 locks(micros) r:248376 nreturned:101 reslen:10423 125ms m31001| Fri Jan 18 13:19:57.240 [conn318] end connection 127.0.0.5:53130 (5 connections now open) m31003| Fri Jan 18 13:19:57.242 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:57.242 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:57.242 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:57.242 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:19:57.242 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.3:31001 m31001| Fri Jan 18 13:19:57.321 [initandlisten] connection accepted from 127.0.0.5:41549 #319 (6 connections now open) m31001| Fri Jan 18 13:19:57.321 [initandlisten] connection accepted from 127.0.0.5:60744 #320 (7 connections now open) m31002| Fri Jan 18 13:19:57.325 [conn144] end connection 127.0.0.5:41030 (5 connections now open) m31001| Fri Jan 18 13:19:57.511 [conn319] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3100 } } cursorid:1145283114850587 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1271 locks(micros) r:200444 nreturned:101 reslen:1737 101ms m31001| Fri Jan 18 13:19:57.587 [conn320] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3100 } } cursorid:1145628109741151 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1271 locks(micros) r:200705 nreturned:101 reslen:10423 101ms m31003| Fri Jan 18 13:19:57.592 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:57.592 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:57.592 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:57.592 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:57.663 [conn320] end connection 127.0.0.5:60744 (6 connections now open) m31001| Fri Jan 18 13:19:57.674 [initandlisten] connection accepted from 127.0.0.5:48796 #321 (7 connections now open) m31001| Fri Jan 18 13:19:58.029 [conn321] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3200 } } cursorid:1147507937800997 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1273 locks(micros) r:246479 nreturned:101 reslen:10423 301ms m31003| Fri Jan 18 13:19:58.032 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:58.032 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:58.032 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:58.032 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:58.034 [conn319] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3100 } } cursorid:1145283114850587 ntoreturn:0 keyUpdates:0 numYields: 672 locks(micros) r:459633 nreturned:246722 reslen:4194294 360ms m31001| Fri Jan 18 13:19:58.064 [conn321] end connection 127.0.0.5:48796 (6 connections now open) m31001| Fri Jan 18 13:19:58.100 [initandlisten] connection accepted from 127.0.0.5:40806 #322 (7 connections now open) m31001| Fri Jan 18 13:19:58.284 [conn322] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3300 } } cursorid:1148591905864325 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1306 locks(micros) r:224800 nreturned:101 reslen:10423 114ms m31003| Fri Jan 18 13:19:58.288 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:58.288 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:58.288 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:58.288 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:58.381 [initandlisten] connection accepted from 127.0.0.5:44914 #323 (8 connections now open) m31001| Fri Jan 18 13:19:58.381 [conn322] end connection 127.0.0.5:40806 (7 connections now open) m31001| Fri Jan 18 13:19:58.585 [conn323] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3400 } } cursorid:1149875292499757 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1305 locks(micros) r:238007 nreturned:101 reslen:10423 120ms m31001| Fri Jan 18 13:19:58.586 [conn323] end connection 127.0.0.5:44914 (6 connections now open) m31003| Fri Jan 18 13:19:58.590 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:58.590 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:58.590 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:19:58.590 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31001| Fri Jan 18 13:19:58.643 [initandlisten] connection accepted from 127.0.0.5:38126 #324 (7 connections now open) m31001| Fri Jan 18 13:19:58.795 [conn324] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3500 } } cursorid:1150767808031648 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1305 locks(micros) r:216908 nreturned:101 reslen:10423 109ms m31003| Fri Jan 18 13:19:58.799 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:58.799 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:58.799 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:19:58.799 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31002| Fri Jan 18 13:19:58.809 [initandlisten] connection accepted from 127.0.0.5:54893 #214 (6 connections now open) m31001| Fri Jan 18 13:19:58.876 [conn324] end connection 127.0.0.5:38126 (6 connections now open) m31002| Fri Jan 18 13:19:59.030 [conn214] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3600 } } cursorid:1149179845356895 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1309 locks(micros) r:244898 nreturned:101 reslen:10423 123ms m31003| Fri Jan 18 13:19:59.032 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 m31001| Fri Jan 18 13:19:59.076 [conn319] end connection 127.0.0.5:41549 (5 connections now open) m31002| Fri Jan 18 13:19:59.081 [conn214] end connection 127.0.0.5:54893 (5 connections now open) m31002| Fri Jan 18 13:19:59.116 [initandlisten] connection accepted from 127.0.0.5:44335 #215 (6 connections now open) m31003| Fri Jan 18 13:19:59.158 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:59.158 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:59.158 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:59.302 [initandlisten] connection accepted from 127.0.0.5:55220 #216 (7 connections now open) m31002| Fri Jan 18 13:19:59.368 [conn215] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3700 } } cursorid:1150632427709587 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1308 locks(micros) r:219957 nreturned:101 reslen:1737 111ms m31002| Fri Jan 18 13:19:59.456 [conn216] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3700 } } cursorid:1151018796404098 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1308 locks(micros) r:209713 nreturned:101 reslen:10423 106ms m31003| Fri Jan 18 13:19:59.459 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:59.459 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:59.459 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:59.479 [conn216] end connection 127.0.0.5:55220 (6 connections now open) m31002| Fri Jan 18 13:19:59.565 [initandlisten] connection accepted from 127.0.0.5:51934 #217 (7 connections now open) m31002| Fri Jan 18 13:19:59.678 [conn217] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3800 } } cursorid:1151967448363470 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1308 locks(micros) r:222132 nreturned:101 reslen:10423 112ms m31003| Fri Jan 18 13:19:59.681 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:19:59.681 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:19:59.681 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:19:59.705 [conn215] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3700 } } cursorid:1150632427709587 ntoreturn:0 keyUpdates:0 numYields: 5 locks(micros) r:368126 nreturned:246722 reslen:4194294 226ms m31002| Fri Jan 18 13:19:59.761 [conn217] end connection 127.0.0.5:51934 (6 connections now open) m31002| Fri Jan 18 13:19:59.850 [initandlisten] connection accepted from 127.0.0.5:48873 #218 (7 connections now open) m31002| Fri Jan 18 13:20:00.015 [conn218] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|3900 } } cursorid:1153393990956659 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1307 locks(micros) r:210363 nreturned:101 reslen:10423 106ms m31003| Fri Jan 18 13:20:00.018 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:00.018 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:00.018 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:00.091 [conn218] end connection 127.0.0.5:48873 (6 connections now open) m31002| Fri Jan 18 13:20:00.103 [initandlisten] connection accepted from 127.0.0.5:55683 #219 (7 connections now open) m31002| Fri Jan 18 13:20:00.224 [conn219] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|4000 } } cursorid:1154298899344282 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1307 locks(micros) r:232673 nreturned:101 reslen:10423 118ms m31003| Fri Jan 18 13:20:00.227 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:00.227 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:00.227 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:00.293 [conn219] end connection 127.0.0.5:55683 (6 connections now open) m31002| Fri Jan 18 13:20:00.293 [initandlisten] connection accepted from 127.0.0.5:48794 #220 (8 connections now open) m31000| Fri Jan 18 13:20:00.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31002| Fri Jan 18 13:20:00.458 [conn220] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|4100 } } cursorid:1155278189771652 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1307 locks(micros) r:224179 nreturned:101 reslen:10423 113ms m31003| Fri Jan 18 13:20:00.460 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:00.460 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:00.460 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:00.500 [conn220] end connection 127.0.0.5:48794 (6 connections now open) m31002| Fri Jan 18 13:20:00.591 [initandlisten] connection accepted from 127.0.0.5:42096 #221 (7 connections now open) m31002| Fri Jan 18 13:20:00.732 [conn221] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|4200 } } cursorid:1156417315615804 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1311 locks(micros) r:256190 nreturned:101 reslen:10423 129ms m31003| Fri Jan 18 13:20:00.734 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:00.734 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:00.734 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:00.770 [conn221] end connection 127.0.0.5:42096 (6 connections now open) m31002| Fri Jan 18 13:20:00.807 [initandlisten] connection accepted from 127.0.0.5:35280 #222 (7 connections now open) m31002| Fri Jan 18 13:20:01.005 [conn222] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|4300 } } cursorid:1157597393284564 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1311 locks(micros) r:210543 nreturned:101 reslen:10423 106ms m31003| Fri Jan 18 13:20:01.008 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:01.008 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:01.008 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:01.009 [conn222] end connection 127.0.0.5:35280 (6 connections now open) m31002| Fri Jan 18 13:20:01.009 [initandlisten] connection accepted from 127.0.0.5:52704 #223 (8 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 274, "optime" : { "t" : 1358533201000, "i" : 155 }, "optimeDate" : ISODate("2013-01-18T18:20:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 240, "optime" : { "t" : 1358533157000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 240, "optime" : { "t" : 1358533157000, "i" : 6083 }, "optimeDate" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:01Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 240, "optime" : { "t" : 1358533199000, "i" : 3342 }, "optimeDate" : ISODate("2013-01-18T18:19:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 239, "optime" : { "t" : 1358533200000, "i" : 3143 }, "optimeDate" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 274, "optime" : { "t" : 1358533160000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:20Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358533157000, "i" : 6083 }, "optimeDate" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:01Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358533015000, "i" : 4100 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358533200000, "i" : 1893 }, "optimeDate" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 239, "optime" : { "t" : 1358533201000, "i" : 50 }, "optimeDate" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358533160000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 273, "optime" : { "t" : 1358533160000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:20Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358533015000, "i" : 4400 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358533201000, "i" : 41 }, "optimeDate" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:01Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 240, "optime" : { "t" : 1358533160000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 46 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 240, "optime" : { "t" : 1358533160000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:01Z"), "pingMs" : 23 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 272, "optime" : { "t" : 1358533015000, "i" : 4400 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 240, "optime" : { "t" : 1358533195000, "i" : 7815 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 239, "optime" : { "t" : 1358533200000, "i" : 2955 }, "optimeDate" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358533160000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358533157000, "i" : 6083 }, "optimeDate" : ISODate("2013-01-18T18:19:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:01Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358533015000, "i" : 4100 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 272, "optime" : { "t" : 1358533201000, "i" : 155 }, "optimeDate" : ISODate("2013-01-18T18:20:01Z"), "self" : true } ], "ok" : 1 } m31002| Fri Jan 18 13:20:01.212 [conn223] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|4400 } } cursorid:1158494693195967 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1310 locks(micros) r:210305 nreturned:101 reslen:10423 106ms m31003| Fri Jan 18 13:20:01.212 [conn1] command admin.$cmd command: { replSetGetStatus: true } ntoreturn:1 keyUpdates:0 reslen:1149 167ms m31002| Fri Jan 18 13:20:01.213 [conn223] end connection 127.0.0.5:52704 (6 connections now open) { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31003| Fri Jan 18 13:20:01.215 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:01.215 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:01.215 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:01.293 [initandlisten] connection accepted from 127.0.0.5:54446 #224 (7 connections now open) m31002| Fri Jan 18 13:20:01.448 [conn224] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|4500 } } cursorid:1159491577377769 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1310 locks(micros) r:221132 nreturned:101 reslen:10423 111ms m31003| Fri Jan 18 13:20:01.453 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:01.453 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:01.453 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:01.518 [conn224] end connection 127.0.0.5:54446 (6 connections now open) m31002| Fri Jan 18 13:20:01.593 [initandlisten] connection accepted from 127.0.0.5:54415 #225 (7 connections now open) m31002| Fri Jan 18 13:20:01.751 [conn225] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|4600 } } cursorid:1160766930492074 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1315 locks(micros) r:208441 nreturned:101 reslen:10423 106ms m31002| Fri Jan 18 13:20:01.751 [conn225] end connection 127.0.0.5:54415 (6 connections now open) m31003| Fri Jan 18 13:20:01.754 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:01.754 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:01.754 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:01.807 [initandlisten] connection accepted from 127.0.0.5:46761 #226 (7 connections now open) m31002| Fri Jan 18 13:20:01.931 [conn226] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|4700 } } cursorid:1161544892236148 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1314 locks(micros) r:209432 nreturned:101 reslen:10423 106ms m31002| Fri Jan 18 13:20:01.931 [conn226] end connection 127.0.0.5:46761 (6 connections now open) m31003| Fri Jan 18 13:20:01.935 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:01.935 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:01.935 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:01.943 [initandlisten] connection accepted from 127.0.0.5:33101 #227 (7 connections now open) m31002| Fri Jan 18 13:20:02.180 [conn227] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|4800 } } cursorid:1162591934530884 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1314 locks(micros) r:217844 nreturned:101 reslen:10423 110ms m31003| Fri Jan 18 13:20:02.184 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:02.184 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:02.184 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:02.189 [conn227] end connection 127.0.0.5:33101 (6 connections now open) m31002| Fri Jan 18 13:20:02.259 [initandlisten] connection accepted from 127.0.0.5:45679 #228 (7 connections now open) m31001| Fri Jan 18 13:20:02.390 [FileAllocator] allocating new datafile /data/db/testReplSet-1/foo.2, filling with zeroes... m31002| Fri Jan 18 13:20:02.517 [conn228] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|4900 } } cursorid:1164022443676859 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1314 locks(micros) r:223481 nreturned:101 reslen:10423 114ms m31003| Fri Jan 18 13:20:02.520 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:02.520 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:02.520 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:02.573 [conn228] end connection 127.0.0.5:45679 (6 connections now open) m31002| Fri Jan 18 13:20:02.573 [initandlisten] connection accepted from 127.0.0.5:52990 #229 (7 connections now open) m31002| Fri Jan 18 13:20:02.769 [conn229] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|5000 } } cursorid:1165100092012172 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1313 locks(micros) r:221052 nreturned:101 reslen:10423 113ms m31003| Fri Jan 18 13:20:02.772 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:02.772 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:02.772 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:02.780 [conn229] end connection 127.0.0.5:52990 (6 connections now open) m31002| Fri Jan 18 13:20:02.821 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.2, filling with zeroes... m31002| Fri Jan 18 13:20:02.835 [initandlisten] connection accepted from 127.0.0.5:38601 #230 (7 connections now open) m31001| Fri Jan 18 13:20:02.915 [FileAllocator] done allocating datafile /data/db/testReplSet-1/foo.2, size: 64MB, took 0.524 secs m31002| Fri Jan 18 13:20:03.348 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.2, size: 64MB, took 0.526 secs m31004| Fri Jan 18 13:20:03.362 [conn33] end connection 127.0.0.2:46697 (4 connections now open) m31004| Fri Jan 18 13:20:03.362 [initandlisten] connection accepted from 127.0.0.2:50502 #37 (5 connections now open) m31002| Fri Jan 18 13:20:03.548 [conn230] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|5100 } } cursorid:1168412883537852 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1329 locks(micros) r:212159 nreturned:101 reslen:10423 108ms m31003| Fri Jan 18 13:20:03.551 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:03.551 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:03.551 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:03.622 [conn230] end connection 127.0.0.5:38601 (6 connections now open) m31002| Fri Jan 18 13:20:03.716 [initandlisten] connection accepted from 127.0.0.5:54526 #231 (7 connections now open) m31002| Fri Jan 18 13:20:03.871 [conn231] query local.oplog.rs query: { ts: { $gte: Timestamp 1358533015000|5200 } } cursorid:1169803721864522 ntoreturn:0 ntoskip:0 nscanned:103 keyUpdates:0 numYields: 1348 locks(micros) r:245063 nreturned:101 reslen:10423 124ms m31002| Fri Jan 18 13:20:03.871 [conn231] end connection 127.0.0.5:54526 (6 connections now open) m31003| Fri Jan 18 13:20:03.873 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:03.873 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:20:03.873 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31002| Fri Jan 18 13:20:03.971 [initandlisten] connection accepted from 127.0.0.5:41231 #232 (7 connections now open) m31003| Fri Jan 18 13:20:04.013 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:04.013 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:20:04.013 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:20:04.013 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31001| Fri Jan 18 13:20:04.080 [initandlisten] connection accepted from 127.0.0.5:46133 #325 (6 connections now open) m31002| Fri Jan 18 13:20:04.108 [conn232] end connection 127.0.0.5:41231 (6 connections now open) m31003| Fri Jan 18 13:20:04.143 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.6:31004 m31003| Fri Jan 18 13:20:04.143 [rsBackgroundSync] SYNCED TO 127.0.0.6:31004 m31003| Fri Jan 18 13:20:04.143 [rsBackgroundSync] replSet syncing to: 127.0.0.6:31004 m31001| Fri Jan 18 13:20:04.235 [conn325] end connection 127.0.0.5:46133 (5 connections now open) m31003| Fri Jan 18 13:20:04.319 [conn31] end connection 127.0.0.6:48938 (3 connections now open) m31003| Fri Jan 18 13:20:04.319 [initandlisten] connection accepted from 127.0.0.6:45312 #34 (4 connections now open) m31004| Fri Jan 18 13:20:04.342 [conn34] end connection 127.0.0.3:50907 (4 connections now open) m31004| Fri Jan 18 13:20:04.342 [initandlisten] connection accepted from 127.0.0.3:35863 #38 (5 connections now open) m31004| Fri Jan 18 13:20:05.010 [conn35] end connection 127.0.0.4:47061 (4 connections now open) m31004| Fri Jan 18 13:20:05.010 [initandlisten] connection accepted from 127.0.0.4:58307 #39 (5 connections now open) m31004| Fri Jan 18 13:20:05.143 [initandlisten] connection accepted from 127.0.0.5:56443 #40 (6 connections now open) m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 600 more seconds m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 599 more seconds m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 599 more seconds m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 600 more seconds m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 599 more seconds m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 599 more seconds m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet error RS102 too stale to catch up, at least from 127.0.0.6:31004 m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet our last optime : Jan 18 13:16:55 50f99197:14b4 m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet oldest at 127.0.0.6:31004 : Jan 18 13:18:29 50f991f5:10f5 m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet error RS102 too stale to catch up m31003| Fri Jan 18 13:20:05.144 [rsBackgroundSync] replSet RECOVERING m31002| Fri Jan 18 13:20:05.363 [conn141] end connection 127.0.0.2:48411 (5 connections now open) m31002| Fri Jan 18 13:20:05.363 [initandlisten] connection accepted from 127.0.0.2:59814 #233 (6 connections now open) m31004| Fri Jan 18 13:20:05.644 [conn40] end connection 127.0.0.5:56443 (5 connections now open) m31003| Fri Jan 18 13:20:05.873 [rsSync] replSet still syncing, not yet to minValid optime 50f991f5:10f5 m31000| Fri Jan 18 13:20:06.151 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:220422 110ms m31001| Fri Jan 18 13:20:06.320 [conn296] end connection 127.0.0.6:42716 (4 connections now open) m31001| Fri Jan 18 13:20:06.320 [initandlisten] connection accepted from 127.0.0.6:53075 #326 (5 connections now open) m31004| Fri Jan 18 13:20:06.320 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state RECOVERING m31002| Fri Jan 18 13:20:06.343 [conn142] end connection 127.0.0.3:41624 (5 connections now open) m31001| Fri Jan 18 13:20:06.343 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state RECOVERING m31002| Fri Jan 18 13:20:06.343 [initandlisten] connection accepted from 127.0.0.3:60584 #234 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 279, "optime" : { "t" : 1358533206000, "i" : 3852 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 245, "optime" : { "t" : 1358533162000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 245, "optime" : { "t" : 1358533162000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 245, "optime" : { "t" : 1358533205000, "i" : 2726 }, "optimeDate" : ISODate("2013-01-18T18:20:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 244, "optime" : { "t" : 1358533206000, "i" : 2383 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 279, "optime" : { "t" : 1358533166000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:26Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 244, "optime" : { "t" : 1358533166000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 244, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 244, "optime" : { "t" : 1358533206000, "i" : 2329 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 244, "optime" : { "t" : 1358533205000, "i" : 64 }, "optimeDate" : ISODate("2013-01-18T18:20:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 244, "optime" : { "t" : 1358533162000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 278, "optime" : { "t" : 1358533166000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:26Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 244, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.6:31004" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 244, "optime" : { "t" : 1358533204000, "i" : 7277 }, "optimeDate" : ISODate("2013-01-18T18:20:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:06Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 245, "optime" : { "t" : 1358533162000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:06Z"), "pingMs" : 42 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 245, "optime" : { "t" : 1358533162000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:05Z"), "pingMs" : 32 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 277, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 245, "optime" : { "t" : 1358533206000, "i" : 282 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 244, "optime" : { "t" : 1358533206000, "i" : 2162 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 244, "optime" : { "t" : 1358533162000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 244, "optime" : { "t" : 1358533166000, "i" : 1681 }, "optimeDate" : ISODate("2013-01-18T18:19:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 244, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 277, "optime" : { "t" : 1358533206000, "i" : 3852 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:20:07.011 [conn300] end connection 127.0.0.4:37685 (4 connections now open) m31001| Fri Jan 18 13:20:07.011 [initandlisten] connection accepted from 127.0.0.4:49056 #327 (5 connections now open) m31002| Fri Jan 18 13:20:07.011 [rsHealthPoll] replSet member 127.0.0.5:31003 is now in state RECOVERING m31001| Fri Jan 18 13:20:09.013 [conn307] end connection 127.0.0.5:52383 (4 connections now open) m31001| Fri Jan 18 13:20:09.078 [initandlisten] connection accepted from 127.0.0.5:37942 #328 (5 connections now open) m31000| Fri Jan 18 13:20:10.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 284, "optime" : { "t" : 1358533211000, "i" : 7351 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 250, "optime" : { "t" : 1358533168000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 250, "optime" : { "t" : 1358533168000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 250, "optime" : { "t" : 1358533211000, "i" : 3173 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 249, "optime" : { "t" : 1358533210000, "i" : 3453 }, "optimeDate" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 284, "optime" : { "t" : 1358533168000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 249, "optime" : { "t" : 1358533168000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:11Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 249, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:11Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 249, "optime" : { "t" : 1358533210000, "i" : 2639 }, "optimeDate" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 249, "optime" : { "t" : 1358533211000, "i" : 13 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 249, "optime" : { "t" : 1358533168000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 283, "optime" : { "t" : 1358533168000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 249, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 249, "optime" : { "t" : 1358533211000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:11Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 250, "optime" : { "t" : 1358533168000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 43 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 250, "optime" : { "t" : 1358533168000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:11Z"), "pingMs" : 16 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 282, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 250, "optime" : { "t" : 1358533211000, "i" : 1389 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 249, "optime" : { "t" : 1358533210000, "i" : 3071 }, "optimeDate" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 249, "optime" : { "t" : 1358533168000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 249, "optime" : { "t" : 1358533168000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:19:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:11Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 249, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:11Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 282, "optime" : { "t" : 1358533211000, "i" : 7351 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31003| Fri Jan 18 13:20:14.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 290, "optime" : { "t" : 1358533217000, "i" : 2527 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358533174000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:16Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358533174000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:17Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:19:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358533215000, "i" : 2579 }, "optimeDate" : ISODate("2013-01-18T18:20:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:16Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 255, "optime" : { "t" : 1358533216000, "i" : 1536 }, "optimeDate" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 290, "optime" : { "t" : 1358533174000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 255, "optime" : { "t" : 1358533174000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:17Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 255, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:17Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 255, "optime" : { "t" : 1358533216000, "i" : 1536 }, "optimeDate" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 255, "optime" : { "t" : 1358533217000, "i" : 126 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 255, "optime" : { "t" : 1358533174000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 289, "optime" : { "t" : 1358533174000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 255, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 255, "optime" : { "t" : 1358533217000, "i" : 100 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:17Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358533174000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 22 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358533174000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:17Z"), "pingMs" : 22 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 288, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358533216000, "i" : 227 }, "optimeDate" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 255, "optime" : { "t" : 1358533216000, "i" : 1536 }, "optimeDate" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 255, "optime" : { "t" : 1358533174000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:16Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 255, "optime" : { "t" : 1358533174000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:17Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 255, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 288, "optime" : { "t" : 1358533217000, "i" : 2527 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31004| Fri Jan 18 13:20:18.627 [conn36] end connection 127.0.0.5:55049 (4 connections now open) m31004| Fri Jan 18 13:20:19.127 [initandlisten] connection accepted from 127.0.0.5:33122 #41 (5 connections now open) m31002| Fri Jan 18 13:20:20.322 [conn188] end connection 127.0.0.6:54525 (5 connections now open) m31002| Fri Jan 18 13:20:20.322 [initandlisten] connection accepted from 127.0.0.6:51486 #235 (6 connections now open) m31003| Fri Jan 18 13:20:20.345 [conn32] end connection 127.0.0.3:38516 (3 connections now open) m31003| Fri Jan 18 13:20:20.345 [initandlisten] connection accepted from 127.0.0.3:43843 #35 (4 connections now open) m31003| Fri Jan 18 13:20:21.013 [conn33] end connection 127.0.0.4:59941 (3 connections now open) m31003| Fri Jan 18 13:20:21.013 [initandlisten] connection accepted from 127.0.0.4:34634 #36 (4 connections now open) m31001| Fri Jan 18 13:20:21.365 [conn317] end connection 127.0.0.2:49978 (4 connections now open) m31001| Fri Jan 18 13:20:21.365 [initandlisten] connection accepted from 127.0.0.2:56701 #329 (5 connections now open) m31000| Fri Jan 18 13:20:21.728 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:1276196 638ms STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:20:22.322 [conn54] end connection 127.0.0.6:59757 (11 connections now open) m31000| Fri Jan 18 13:20:22.322 [initandlisten] connection accepted from 127.0.0.6:51006 #57 (13 connections now open) m31000| Fri Jan 18 13:20:22.345 [conn55] end connection 127.0.0.3:41117 (11 connections now open) m31000| Fri Jan 18 13:20:22.345 [initandlisten] connection accepted from 127.0.0.3:56596 #58 (13 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 295, "optime" : { "t" : 1358533222000, "i" : 8561 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 261, "optime" : { "t" : 1358533179000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 261, "optime" : { "t" : 1358533179000, "i" : 907 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 261, "optime" : { "t" : 1358533221000, "i" : 977 }, "optimeDate" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 260, "optime" : { "t" : 1358533222000, "i" : 5416 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 295, "optime" : { "t" : 1358533182000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:42Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 260, "optime" : { "t" : 1358533179000, "i" : 907 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 260, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 260, "optime" : { "t" : 1358533222000, "i" : 5416 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 260, "optime" : { "t" : 1358533221000, "i" : 206 }, "optimeDate" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:21Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 260, "optime" : { "t" : 1358533179000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 294, "optime" : { "t" : 1358533182000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:42Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 260, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:21Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 260, "optime" : { "t" : 1358533220000, "i" : 5579 }, "optimeDate" : ISODate("2013-01-18T18:20:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:22Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 261, "optime" : { "t" : 1358533179000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:22Z"), "pingMs" : 17 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 261, "optime" : { "t" : 1358533179000, "i" : 907 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 22 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 293, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 261, "optime" : { "t" : 1358533221000, "i" : 977 }, "optimeDate" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 260, "optime" : { "t" : 1358533222000, "i" : 5416 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 260, "optime" : { "t" : 1358533179000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 260, "optime" : { "t" : 1358533179000, "i" : 907 }, "optimeDate" : ISODate("2013-01-18T18:19:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 260, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 293, "optime" : { "t" : 1358533222000, "i" : 6543 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31000| Fri Jan 18 13:20:23.013 [conn56] end connection 127.0.0.4:55347 (11 connections now open) m31000| Fri Jan 18 13:20:23.013 [initandlisten] connection accepted from 127.0.0.4:35410 #59 (12 connections now open) m31002| Fri Jan 18 13:20:23.053 [conn197] end connection 127.0.0.5:56602 (5 connections now open) m31002| Fri Jan 18 13:20:23.102 [initandlisten] connection accepted from 127.0.0.5:43804 #236 (6 connections now open) m31003| Fri Jan 18 13:20:24.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Fri Jan 18 13:20:27.485 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:443611 221ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 300, "optime" : { "t" : 1358533227000, "i" : 5922 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 266, "optime" : { "t" : 1358533185000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 266, "optime" : { "t" : 1358533184000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 266, "optime" : { "t" : 1358533227000, "i" : 2070 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 265, "optime" : { "t" : 1358533226000, "i" : 1404 }, "optimeDate" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 300, "optime" : { "t" : 1358533185000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:45Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 265, "optime" : { "t" : 1358533184000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:27Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 265, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:27Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 265, "optime" : { "t" : 1358533226000, "i" : 1404 }, "optimeDate" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 265, "optime" : { "t" : 1358533226000, "i" : 9441 }, "optimeDate" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 265, "optime" : { "t" : 1358533185000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 299, "optime" : { "t" : 1358533184000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:44Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 265, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:27Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 265, "optime" : { "t" : 1358533226000, "i" : 9441 }, "optimeDate" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:27Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 266, "optime" : { "t" : 1358533185000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 266, "optime" : { "t" : 1358533184000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 26 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 298, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 266, "optime" : { "t" : 1358533226000, "i" : 5291 }, "optimeDate" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 265, "optime" : { "t" : 1358533226000, "i" : 1404 }, "optimeDate" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 265, "optime" : { "t" : 1358533185000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:26Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 265, "optime" : { "t" : 1358533184000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:19:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:27Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 265, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:26Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 298, "optime" : { "t" : 1358533227000, "i" : 5730 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31000| Fri Jan 18 13:20:31.312 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:243315 121ms m31000| Fri Jan 18 13:20:32.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 305, "optime" : { "t" : 1358533232000, "i" : 8093 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 271, "optime" : { "t" : 1358533190000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 271, "optime" : { "t" : 1358533190000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 271, "optime" : { "t" : 1358533231000, "i" : 3215 }, "optimeDate" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:32Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 270, "optime" : { "t" : 1358533232000, "i" : 3235 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 305, "optime" : { "t" : 1358533190000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 270, "optime" : { "t" : 1358533190000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 270, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 270, "optime" : { "t" : 1358533232000, "i" : 3230 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 270, "optime" : { "t" : 1358533231000, "i" : 83 }, "optimeDate" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 270, "optime" : { "t" : 1358533190000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 304, "optime" : { "t" : 1358533190000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 270, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:31Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 270, "optime" : { "t" : 1358533231000, "i" : 17 }, "optimeDate" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:32Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:32Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 271, "optime" : { "t" : 1358533190000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:32Z"), "pingMs" : 20 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 271, "optime" : { "t" : 1358533190000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 24 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 303, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 271, "optime" : { "t" : 1358533231000, "i" : 5661 }, "optimeDate" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 270, "optime" : { "t" : 1358533232000, "i" : 3060 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 270, "optime" : { "t" : 1358533190000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 270, "optime" : { "t" : 1358533190000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:19:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 270, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 303, "optime" : { "t" : 1358533232000, "i" : 8093 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31004| Fri Jan 18 13:20:33.367 [conn37] end connection 127.0.0.2:50502 (4 connections now open) m31004| Fri Jan 18 13:20:33.367 [initandlisten] connection accepted from 127.0.0.2:45719 #42 (5 connections now open) m31000| Fri Jan 18 13:20:34.061 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:170808 nreturned:160000 reslen:2720020 136ms m31000| Fri Jan 18 13:20:34.075 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:150817 nreturned:160000 reslen:2720020 133ms m31003| Fri Jan 18 13:20:34.324 [conn34] end connection 127.0.0.6:45312 (3 connections now open) m31003| Fri Jan 18 13:20:34.324 [initandlisten] connection accepted from 127.0.0.6:48564 #37 (5 connections now open) m31004| Fri Jan 18 13:20:34.347 [conn38] end connection 127.0.0.3:35863 (4 connections now open) m31004| Fri Jan 18 13:20:34.347 [initandlisten] connection accepted from 127.0.0.3:48616 #43 (5 connections now open) m31004| Fri Jan 18 13:20:35.015 [conn39] end connection 127.0.0.4:58307 (4 connections now open) m31004| Fri Jan 18 13:20:35.015 [initandlisten] connection accepted from 127.0.0.4:56201 #44 (5 connections now open) m31002| Fri Jan 18 13:20:35.367 [conn233] end connection 127.0.0.2:59814 (5 connections now open) m31002| Fri Jan 18 13:20:35.367 [initandlisten] connection accepted from 127.0.0.2:33923 #237 (6 connections now open) m31001| Fri Jan 18 13:20:36.324 [conn326] end connection 127.0.0.6:53075 (4 connections now open) m31001| Fri Jan 18 13:20:36.324 [initandlisten] connection accepted from 127.0.0.6:58091 #330 (5 connections now open) m31002| Fri Jan 18 13:20:36.347 [conn234] end connection 127.0.0.3:60584 (5 connections now open) m31002| Fri Jan 18 13:20:36.347 [initandlisten] connection accepted from 127.0.0.3:41974 #238 (7 connections now open) m31001| Fri Jan 18 13:20:37.015 [conn327] end connection 127.0.0.4:49056 (4 connections now open) m31001| Fri Jan 18 13:20:37.015 [initandlisten] connection accepted from 127.0.0.4:48328 #331 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 311, "optime" : { "t" : 1358533238000, "i" : 2435 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 277, "optime" : { "t" : 1358533195000, "i" : 6603 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 277, "optime" : { "t" : 1358533195000, "i" : 6603 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 277, "optime" : { "t" : 1358533237000, "i" : 2784 }, "optimeDate" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 276, "optime" : { "t" : 1358533236000, "i" : 2971 }, "optimeDate" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:37Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 311, "optime" : { "t" : 1358533195000, "i" : 6603 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 276, "optime" : { "t" : 1358533195000, "i" : 6603 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:37Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 276, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:37Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 276, "optime" : { "t" : 1358533236000, "i" : 2971 }, "optimeDate" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 276, "optime" : { "t" : 1358533237000, "i" : 86 }, "optimeDate" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:37Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 276, "optime" : { "t" : 1358533195000, "i" : 6603 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 310, "optime" : { "t" : 1358533195000, "i" : 6603 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 276, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:37Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 276, "optime" : { "t" : 1358533236000, "i" : 8818 }, "optimeDate" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:38Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 277, "optime" : { "t" : 1358533195000, "i" : 6603 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 19 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 277, "optime" : { "t" : 1358533195000, "i" : 6603 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 12 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 309, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 277, "optime" : { "t" : 1358533236000, "i" : 5302 }, "optimeDate" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 276, "optime" : { "t" : 1358533236000, "i" : 2872 }, "optimeDate" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:37Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 276, "optime" : { "t" : 1358533195000, "i" : 6603 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:36Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 276, "optime" : { "t" : 1358533195000, "i" : 6603 }, "optimeDate" : ISODate("2013-01-18T18:19:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:37Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 276, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:36Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 309, "optime" : { "t" : 1358533238000, "i" : 2435 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:20:39.361 [conn328] end connection 127.0.0.5:37942 (4 connections now open) m31001| Fri Jan 18 13:20:39.393 [initandlisten] connection accepted from 127.0.0.5:35289 #332 (5 connections now open) m31000| Fri Jan 18 13:20:42.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 316, "optime" : { "t" : 1358533243000, "i" : 5087 }, "optimeDate" : ISODate("2013-01-18T18:20:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 282, "optime" : { "t" : 1358533200000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 282, "optime" : { "t" : 1358533201000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 282, "optime" : { "t" : 1358533243000, "i" : 3361 }, "optimeDate" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 281, "optime" : { "t" : 1358533242000, "i" : 3304 }, "optimeDate" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 316, "optime" : { "t" : 1358533200000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:00Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 281, "optime" : { "t" : 1358533201000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:43Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 281, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 281, "optime" : { "t" : 1358533242000, "i" : 2943 }, "optimeDate" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 281, "optime" : { "t" : 1358533243000, "i" : 117 }, "optimeDate" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 281, "optime" : { "t" : 1358533200000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 315, "optime" : { "t" : 1358533201000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:01Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 281, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:43Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 281, "optime" : { "t" : 1358533243000, "i" : 25 }, "optimeDate" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:43Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 282, "optime" : { "t" : 1358533200000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:42Z"), "pingMs" : 40 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 282, "optime" : { "t" : 1358533201000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 20 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 314, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 282, "optime" : { "t" : 1358533241000, "i" : 2962 }, "optimeDate" : ISODate("2013-01-18T18:20:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 281, "optime" : { "t" : 1358533242000, "i" : 3084 }, "optimeDate" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 281, "optime" : { "t" : 1358533200000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:42Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 281, "optime" : { "t" : 1358533201000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:43Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 281, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 314, "optime" : { "t" : 1358533243000, "i" : 5087 }, "optimeDate" : ISODate("2013-01-18T18:20:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31003| Fri Jan 18 13:20:46.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 321, "optime" : { "t" : 1358533248000, "i" : 6628 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 287, "optime" : { "t" : 1358533206000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 287, "optime" : { "t" : 1358533206000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 287, "optime" : { "t" : 1358533247000, "i" : 4370 }, "optimeDate" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:48Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 286, "optime" : { "t" : 1358533248000, "i" : 2751 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 321, "optime" : { "t" : 1358533206000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 286, "optime" : { "t" : 1358533206000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 286, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 286, "optime" : { "t" : 1358533248000, "i" : 2739 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 286, "optime" : { "t" : 1358533247000, "i" : 381 }, "optimeDate" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 286, "optime" : { "t" : 1358533206000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 320, "optime" : { "t" : 1358533206000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 286, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:47Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 286, "optime" : { "t" : 1358533247000, "i" : 254 }, "optimeDate" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:48Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:48Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 287, "optime" : { "t" : 1358533206000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:48Z"), "pingMs" : 38 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 287, "optime" : { "t" : 1358533206000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 16 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 319, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 287, "optime" : { "t" : 1358533246000, "i" : 260 }, "optimeDate" : ISODate("2013-01-18T18:20:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 286, "optime" : { "t" : 1358533248000, "i" : 2581 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 286, "optime" : { "t" : 1358533206000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 286, "optime" : { "t" : 1358533206000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 286, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 319, "optime" : { "t" : 1358533248000, "i" : 6628 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31002| Fri Jan 18 13:20:50.326 [conn235] end connection 127.0.0.6:51486 (5 connections now open) m31002| Fri Jan 18 13:20:50.326 [initandlisten] connection accepted from 127.0.0.6:33169 #239 (6 connections now open) m31003| Fri Jan 18 13:20:50.349 [conn35] end connection 127.0.0.3:43843 (3 connections now open) m31003| Fri Jan 18 13:20:50.349 [initandlisten] connection accepted from 127.0.0.3:39592 #38 (4 connections now open) m31003| Fri Jan 18 13:20:51.017 [conn36] end connection 127.0.0.4:34634 (3 connections now open) m31003| Fri Jan 18 13:20:51.017 [initandlisten] connection accepted from 127.0.0.4:49033 #39 (4 connections now open) m31001| Fri Jan 18 13:20:51.370 [conn329] end connection 127.0.0.2:56701 (4 connections now open) m31001| Fri Jan 18 13:20:51.370 [initandlisten] connection accepted from 127.0.0.2:56037 #333 (5 connections now open) m31000| Fri Jan 18 13:20:52.326 [conn57] end connection 127.0.0.6:51006 (11 connections now open) m31000| Fri Jan 18 13:20:52.327 [initandlisten] connection accepted from 127.0.0.6:54952 #60 (12 connections now open) m31000| Fri Jan 18 13:20:52.349 [conn58] end connection 127.0.0.3:56596 (11 connections now open) m31000| Fri Jan 18 13:20:52.349 [initandlisten] connection accepted from 127.0.0.3:33464 #61 (12 connections now open) m31000| Fri Jan 18 13:20:53.017 [conn59] end connection 127.0.0.4:35410 (11 connections now open) m31000| Fri Jan 18 13:20:53.017 [initandlisten] connection accepted from 127.0.0.4:39524 #62 (12 connections now open) m31002| Fri Jan 18 13:20:53.391 [conn236] end connection 127.0.0.5:43804 (5 connections now open) m31002| Fri Jan 18 13:20:53.432 [initandlisten] connection accepted from 127.0.0.5:45659 #240 (6 connections now open) m31003| Fri Jan 18 13:20:54.132 [rsMgr] replSet I don't see a primary and I can't elect myself STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 327, "optime" : { "t" : 1358533254000, "i" : 1673 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 293, "optime" : { "t" : 1358533211000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 293, "optime" : { "t" : 1358533211000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 293, "optime" : { "t" : 1358533253000, "i" : 3399 }, "optimeDate" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 292, "optime" : { "t" : 1358533252000, "i" : 3093 }, "optimeDate" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:53Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 327, "optime" : { "t" : 1358533211000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 292, "optime" : { "t" : 1358533211000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:53Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 292, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:53Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 292, "optime" : { "t" : 1358533252000, "i" : 3074 }, "optimeDate" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 292, "optime" : { "t" : 1358533253000, "i" : 297 }, "optimeDate" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:53Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 292, "optime" : { "t" : 1358533211000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 326, "optime" : { "t" : 1358533211000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 292, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:53Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 292, "optime" : { "t" : 1358533253000, "i" : 255 }, "optimeDate" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:54Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 293, "optime" : { "t" : 1358533211000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 19 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 293, "optime" : { "t" : 1358533211000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 17 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 325, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 293, "optime" : { "t" : 1358533254000, "i" : 1061 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 292, "optime" : { "t" : 1358533252000, "i" : 2802 }, "optimeDate" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:53Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 292, "optime" : { "t" : 1358533211000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:52Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 292, "optime" : { "t" : 1358533211000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:20:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:53Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 292, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:54Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 325, "optime" : { "t" : 1358533254000, "i" : 1673 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31003| Fri Jan 18 13:20:56.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31004| Fri Jan 18 13:20:56.632 [conn41] end connection 127.0.0.5:33122 (4 connections now open) m31004| Fri Jan 18 13:20:57.132 [initandlisten] connection accepted from 127.0.0.5:58216 #45 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 332, "optime" : { "t" : 1358533259000, "i" : 3889 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 298, "optime" : { "t" : 1358533217000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 298, "optime" : { "t" : 1358533217000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 298, "optime" : { "t" : 1358533259000, "i" : 3197 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 297, "optime" : { "t" : 1358533258000, "i" : 3004 }, "optimeDate" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 332, "optime" : { "t" : 1358533217000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 297, "optime" : { "t" : 1358533217000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:59Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 297, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 297, "optime" : { "t" : 1358533258000, "i" : 2984 }, "optimeDate" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 297, "optime" : { "t" : 1358533258000, "i" : 8790 }, "optimeDate" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 297, "optime" : { "t" : 1358533217000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 331, "optime" : { "t" : 1358533217000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 297, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:59Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 297, "optime" : { "t" : 1358533258000, "i" : 8790 }, "optimeDate" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:59Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 298, "optime" : { "t" : 1358533217000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:58Z"), "pingMs" : 37 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 298, "optime" : { "t" : 1358533217000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 16 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 330, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 298, "optime" : { "t" : 1358533254000, "i" : 1673 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:20:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 297, "optime" : { "t" : 1358533258000, "i" : 2539 }, "optimeDate" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 297, "optime" : { "t" : 1358533217000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:58Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 297, "optime" : { "t" : 1358533217000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:20:59Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 297, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 330, "optime" : { "t" : 1358533259000, "i" : 3889 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31004| Fri Jan 18 13:21:03.371 [conn42] end connection 127.0.0.2:45719 (4 connections now open) m31004| Fri Jan 18 13:21:03.372 [initandlisten] connection accepted from 127.0.0.2:53723 #46 (5 connections now open) m31003| Fri Jan 18 13:21:04.328 [conn37] end connection 127.0.0.6:48564 (3 connections now open) m31003| Fri Jan 18 13:21:04.328 [initandlisten] connection accepted from 127.0.0.6:60269 #40 (4 connections now open) m31000| Fri Jan 18 13:21:04.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31004| Fri Jan 18 13:21:04.351 [conn43] end connection 127.0.0.3:48616 (4 connections now open) m31004| Fri Jan 18 13:21:04.351 [initandlisten] connection accepted from 127.0.0.3:56958 #47 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 337, "optime" : { "t" : 1358533264000, "i" : 6868 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 303, "optime" : { "t" : 1358533222000, "i" : 4269 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:04Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 303, "optime" : { "t" : 1358533222000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 303, "optime" : { "t" : 1358533263000, "i" : 2863 }, "optimeDate" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:04Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 302, "optime" : { "t" : 1358533264000, "i" : 2739 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 337, "optime" : { "t" : 1358533222000, "i" : 4269 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 302, "optime" : { "t" : 1358533222000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 302, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 302, "optime" : { "t" : 1358533264000, "i" : 2739 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 302, "optime" : { "t" : 1358533263000, "i" : 52 }, "optimeDate" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:03Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 302, "optime" : { "t" : 1358533222000, "i" : 4269 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:04Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 336, "optime" : { "t" : 1358533222000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 302, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:03Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 302, "optime" : { "t" : 1358533263000, "i" : 52 }, "optimeDate" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:04Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:04Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 303, "optime" : { "t" : 1358533222000, "i" : 4269 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:04Z"), "pingMs" : 46 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 303, "optime" : { "t" : 1358533222000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 19 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 335, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 303, "optime" : { "t" : 1358533264000, "i" : 5430 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 302, "optime" : { "t" : 1358533264000, "i" : 2739 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 302, "optime" : { "t" : 1358533222000, "i" : 4269 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:04Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 302, "optime" : { "t" : 1358533222000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 302, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:04Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 335, "optime" : { "t" : 1358533264000, "i" : 6846 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31004| Fri Jan 18 13:21:05.019 [conn44] end connection 127.0.0.4:56201 (4 connections now open) m31004| Fri Jan 18 13:21:05.019 [initandlisten] connection accepted from 127.0.0.4:57277 #48 (6 connections now open) m31002| Fri Jan 18 13:21:05.372 [conn237] end connection 127.0.0.2:33923 (5 connections now open) m31002| Fri Jan 18 13:21:05.372 [initandlisten] connection accepted from 127.0.0.2:48595 #241 (6 connections now open) m31003| Fri Jan 18 13:21:05.880 [rsSync] replSet still syncing, not yet to minValid optime 50f991f5:10f5 m31001| Fri Jan 18 13:21:06.328 [conn330] end connection 127.0.0.6:58091 (4 connections now open) m31001| Fri Jan 18 13:21:06.329 [initandlisten] connection accepted from 127.0.0.6:35353 #334 (5 connections now open) m31002| Fri Jan 18 13:21:06.351 [conn238] end connection 127.0.0.3:41974 (5 connections now open) m31002| Fri Jan 18 13:21:06.351 [initandlisten] connection accepted from 127.0.0.3:52432 #242 (6 connections now open) m31001| Fri Jan 18 13:21:07.019 [conn331] end connection 127.0.0.4:48328 (4 connections now open) m31001| Fri Jan 18 13:21:07.019 [initandlisten] connection accepted from 127.0.0.4:56920 #335 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31001| Fri Jan 18 13:21:10.170 [conn332] end connection 127.0.0.5:35289 (4 connections now open) m31001| Fri Jan 18 13:21:10.170 [initandlisten] connection accepted from 127.0.0.5:59276 #336 (5 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 343, "optime" : { "t" : 1358533270000, "i" : 1839 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 309, "optime" : { "t" : 1358533227000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 309, "optime" : { "t" : 1358533227000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 309, "optime" : { "t" : 1358533269000, "i" : 2876 }, "optimeDate" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 308, "optime" : { "t" : 1358533268000, "i" : 3127 }, "optimeDate" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 343, "optime" : { "t" : 1358533227000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 308, "optime" : { "t" : 1358533227000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:09Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 308, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:10Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 308, "optime" : { "t" : 1358533268000, "i" : 3122 }, "optimeDate" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 308, "optime" : { "t" : 1358533269000, "i" : 121 }, "optimeDate" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 308, "optime" : { "t" : 1358533227000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 342, "optime" : { "t" : 1358533230000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:30Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 308, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:09Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 308, "optime" : { "t" : 1358533269000, "i" : 109 }, "optimeDate" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:10Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 309, "optime" : { "t" : 1358533227000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 44 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 309, "optime" : { "t" : 1358533227000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 16 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 341, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 309, "optime" : { "t" : 1358533269000, "i" : 5330 }, "optimeDate" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 308, "optime" : { "t" : 1358533268000, "i" : 2824 }, "optimeDate" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 308, "optime" : { "t" : 1358533227000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 308, "optime" : { "t" : 1358533227000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:09Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 308, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:09Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 341, "optime" : { "t" : 1358533270000, "i" : 1669 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31000| Fri Jan 18 13:21:14.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 348, "optime" : { "t" : 1358533275000, "i" : 4622 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 314, "optime" : { "t" : 1358533232000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 314, "optime" : { "t" : 1358533232000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 314, "optime" : { "t" : 1358533275000, "i" : 2733 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 313, "optime" : { "t" : 1358533274000, "i" : 3261 }, "optimeDate" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 348, "optime" : { "t" : 1358533232000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 313, "optime" : { "t" : 1358533232000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:15Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 313, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 313, "optime" : { "t" : 1358533274000, "i" : 3234 }, "optimeDate" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 313, "optime" : { "t" : 1358533275000, "i" : 103 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 313, "optime" : { "t" : 1358533232000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 347, "optime" : { "t" : 1358533232000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 313, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 313, "optime" : { "t" : 1358533274000, "i" : 8928 }, "optimeDate" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:15Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 314, "optime" : { "t" : 1358533232000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:14Z"), "pingMs" : 40 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 314, "optime" : { "t" : 1358533232000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:15Z"), "pingMs" : 19 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 346, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 314, "optime" : { "t" : 1358533274000, "i" : 5719 }, "optimeDate" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 313, "optime" : { "t" : 1358533274000, "i" : 3046 }, "optimeDate" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 313, "optime" : { "t" : 1358533232000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:14Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 313, "optime" : { "t" : 1358533232000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:15Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 313, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:14Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 346, "optime" : { "t" : 1358533275000, "i" : 4622 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31000| Fri Jan 18 13:21:16.089 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:137811 nreturned:160000 reslen:2720020 137ms m31000| Fri Jan 18 13:21:16.124 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:132114 nreturned:160000 reslen:2720020 132ms m31003| Fri Jan 18 13:21:18.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Fri Jan 18 13:21:20.331 [conn239] end connection 127.0.0.6:33169 (5 connections now open) m31002| Fri Jan 18 13:21:20.331 [initandlisten] connection accepted from 127.0.0.6:53589 #243 (7 connections now open) m31003| Fri Jan 18 13:21:20.353 [conn38] end connection 127.0.0.3:39592 (3 connections now open) m31003| Fri Jan 18 13:21:20.354 [initandlisten] connection accepted from 127.0.0.3:47129 #41 (4 connections now open) STATUS FROM : 127.0.0.2:31000 m31003| Fri Jan 18 13:21:21.021 [conn39] end connection 127.0.0.4:49033 (3 connections now open) m31003| Fri Jan 18 13:21:21.021 [initandlisten] connection accepted from 127.0.0.4:54411 #42 (4 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 354, "optime" : { "t" : 1358533281000, "i" : 400 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 320, "optime" : { "t" : 1358533238000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:20Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 320, "optime" : { "t" : 1358533238000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:21Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:20:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 320, "optime" : { "t" : 1358533279000, "i" : 2917 }, "optimeDate" : ISODate("2013-01-18T18:21:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:20Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 319, "optime" : { "t" : 1358533280000, "i" : 2560 }, "optimeDate" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 354, "optime" : { "t" : 1358533238000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 319, "optime" : { "t" : 1358533238000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:21Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 319, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 319, "optime" : { "t" : 1358533280000, "i" : 2517 }, "optimeDate" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 319, "optime" : { "t" : 1358533281000, "i" : 384 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 319, "optime" : { "t" : 1358533238000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 353, "optime" : { "t" : 1358533238000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 319, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 319, "optime" : { "t" : 1358533281000, "i" : 384 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:21Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 320, "optime" : { "t" : 1358533238000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:20Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 320, "optime" : { "t" : 1358533238000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:21Z"), "pingMs" : 19 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 352, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 320, "optime" : { "t" : 1358533279000, "i" : 5055 }, "optimeDate" : ISODate("2013-01-18T18:21:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 319, "optime" : { "t" : 1358533280000, "i" : 2409 }, "optimeDate" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 319, "optime" : { "t" : 1358533238000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:20Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 319, "optime" : { "t" : 1358533238000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:21Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 319, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 352, "optime" : { "t" : 1358533281000, "i" : 400 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:21:21.374 [conn333] end connection 127.0.0.2:56037 (4 connections now open) m31001| Fri Jan 18 13:21:21.374 [initandlisten] connection accepted from 127.0.0.2:35462 #337 (6 connections now open) m31000| Fri Jan 18 13:21:22.331 [conn60] end connection 127.0.0.6:54952 (11 connections now open) m31000| Fri Jan 18 13:21:22.331 [initandlisten] connection accepted from 127.0.0.6:41941 #63 (13 connections now open) m31000| Fri Jan 18 13:21:22.353 [conn61] end connection 127.0.0.3:33464 (11 connections now open) m31000| Fri Jan 18 13:21:22.354 [initandlisten] connection accepted from 127.0.0.3:43533 #64 (12 connections now open) m31000| Fri Jan 18 13:21:23.022 [conn62] end connection 127.0.0.4:39524 (11 connections now open) m31000| Fri Jan 18 13:21:23.022 [initandlisten] connection accepted from 127.0.0.4:40949 #65 (12 connections now open) m31002| Fri Jan 18 13:21:23.763 [conn240] end connection 127.0.0.5:45659 (5 connections now open) m31002| Fri Jan 18 13:21:23.781 [initandlisten] connection accepted from 127.0.0.5:58593 #244 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 359, "optime" : { "t" : 1358533286000, "i" : 2992 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 325, "optime" : { "t" : 1358533243000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:26Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 325, "optime" : { "t" : 1358533243000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 325, "optime" : { "t" : 1358533285000, "i" : 3313 }, "optimeDate" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:26Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 324, "optime" : { "t" : 1358533286000, "i" : 1273 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 359, "optime" : { "t" : 1358533246000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:46Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 324, "optime" : { "t" : 1358533246000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 324, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 324, "optime" : { "t" : 1358533286000, "i" : 1273 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 324, "optime" : { "t" : 1358533285000, "i" : 182 }, "optimeDate" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:25Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 324, "optime" : { "t" : 1358533243000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:26Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 358, "optime" : { "t" : 1358533246000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:46Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 324, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:25Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 324, "optime" : { "t" : 1358533285000, "i" : 127 }, "optimeDate" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:26Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:26Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 325, "optime" : { "t" : 1358533246000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:26Z"), "pingMs" : 19 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 325, "optime" : { "t" : 1358533243000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 20 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 357, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 325, "optime" : { "t" : 1358533284000, "i" : 4850 }, "optimeDate" : ISODate("2013-01-18T18:21:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 324, "optime" : { "t" : 1358533286000, "i" : 1273 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 324, "optime" : { "t" : 1358533246000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:26Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 324, "optime" : { "t" : 1358533246000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 324, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 357, "optime" : { "t" : 1358533286000, "i" : 2992 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31003| Fri Jan 18 13:21:28.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Fri Jan 18 13:21:30.021 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:410671 205ms m31000| Fri Jan 18 13:21:30.304 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:359286 179ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 364, "optime" : { "t" : 1358533291000, "i" : 5854 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 330, "optime" : { "t" : 1358533248000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 330, "optime" : { "t" : 1358533251000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 330, "optime" : { "t" : 1358533291000, "i" : 3198 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 329, "optime" : { "t" : 1358533290000, "i" : 3852 }, "optimeDate" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 364, "optime" : { "t" : 1358533248000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 329, "optime" : { "t" : 1358533248000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:31Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 329, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:30Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 329, "optime" : { "t" : 1358533290000, "i" : 3787 }, "optimeDate" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 329, "optime" : { "t" : 1358533291000, "i" : 143 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 329, "optime" : { "t" : 1358533248000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 363, "optime" : { "t" : 1358533251000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:51Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 329, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 329, "optime" : { "t" : 1358533291000, "i" : 128 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:31Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 330, "optime" : { "t" : 1358533248000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 20 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 330, "optime" : { "t" : 1358533248000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:31Z"), "pingMs" : 20 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 362, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 330, "optime" : { "t" : 1358533289000, "i" : 327 }, "optimeDate" : ISODate("2013-01-18T18:21:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 329, "optime" : { "t" : 1358533290000, "i" : 3216 }, "optimeDate" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 329, "optime" : { "t" : 1358533248000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:30Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 329, "optime" : { "t" : 1358533248000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:20:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:31Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 329, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 362, "optime" : { "t" : 1358533291000, "i" : 5854 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31004| Fri Jan 18 13:21:33.376 [conn46] end connection 127.0.0.2:53723 (4 connections now open) m31004| Fri Jan 18 13:21:33.376 [initandlisten] connection accepted from 127.0.0.2:37988 #49 (5 connections now open) m31003| Fri Jan 18 13:21:34.333 [conn40] end connection 127.0.0.6:60269 (3 connections now open) m31003| Fri Jan 18 13:21:34.333 [initandlisten] connection accepted from 127.0.0.6:42211 #43 (5 connections now open) m31004| Fri Jan 18 13:21:34.356 [conn47] end connection 127.0.0.3:56958 (4 connections now open) m31004| Fri Jan 18 13:21:34.356 [initandlisten] connection accepted from 127.0.0.3:54858 #50 (6 connections now open) m31004| Fri Jan 18 13:21:34.637 [conn45] end connection 127.0.0.5:58216 (4 connections now open) m31004| Fri Jan 18 13:21:35.023 [conn48] end connection 127.0.0.4:57277 (3 connections now open) m31004| Fri Jan 18 13:21:35.023 [initandlisten] connection accepted from 127.0.0.4:60626 #51 (4 connections now open) m31004| Fri Jan 18 13:21:35.137 [initandlisten] connection accepted from 127.0.0.5:53328 #52 (5 connections now open) m31002| Fri Jan 18 13:21:35.376 [conn241] end connection 127.0.0.2:48595 (5 connections now open) m31002| Fri Jan 18 13:21:35.376 [initandlisten] connection accepted from 127.0.0.2:55487 #245 (6 connections now open) m31000| Fri Jan 18 13:21:36.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31001| Fri Jan 18 13:21:36.333 [conn334] end connection 127.0.0.6:35353 (4 connections now open) m31001| Fri Jan 18 13:21:36.333 [initandlisten] connection accepted from 127.0.0.6:60595 #338 (5 connections now open) m31002| Fri Jan 18 13:21:36.356 [conn242] end connection 127.0.0.3:52432 (5 connections now open) m31002| Fri Jan 18 13:21:36.356 [initandlisten] connection accepted from 127.0.0.3:38868 #246 (7 connections now open) m31001| Fri Jan 18 13:21:37.023 [conn335] end connection 127.0.0.4:56920 (4 connections now open) m31001| Fri Jan 18 13:21:37.024 [initandlisten] connection accepted from 127.0.0.4:42642 #339 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 370, "optime" : { "t" : 1358533297000, "i" : 184 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 336, "optime" : { "t" : 1358533254000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:36Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 336, "optime" : { "t" : 1358533254000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:37Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 336, "optime" : { "t" : 1358533295000, "i" : 2868 }, "optimeDate" : ISODate("2013-01-18T18:21:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:36Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 335, "optime" : { "t" : 1358533296000, "i" : 2722 }, "optimeDate" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 370, "optime" : { "t" : 1358533254000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 335, "optime" : { "t" : 1358533254000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:37Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 335, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:36Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 335, "optime" : { "t" : 1358533296000, "i" : 2722 }, "optimeDate" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 335, "optime" : { "t" : 1358533297000, "i" : 151 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 335, "optime" : { "t" : 1358533254000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 369, "optime" : { "t" : 1358533254000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 335, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 335, "optime" : { "t" : 1358533297000, "i" : 9 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:37Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 336, "optime" : { "t" : 1358533254000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 52 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 336, "optime" : { "t" : 1358533254000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:37Z"), "pingMs" : 24 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 368, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 336, "optime" : { "t" : 1358533295000, "i" : 646 }, "optimeDate" : ISODate("2013-01-18T18:21:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 335, "optime" : { "t" : 1358533296000, "i" : 2683 }, "optimeDate" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 335, "optime" : { "t" : 1358533254000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:36Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 335, "optime" : { "t" : 1358533254000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:37Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 335, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 368, "optime" : { "t" : 1358533297000, "i" : 9 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31001| Fri Jan 18 13:21:40.800 [conn336] end connection 127.0.0.5:59276 (4 connections now open) m31001| Fri Jan 18 13:21:40.828 [initandlisten] connection accepted from 127.0.0.5:41284 #340 (5 connections now open) m31000| Fri Jan 18 13:21:41.325 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:500045 250ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 375, "optime" : { "t" : 1358533302000, "i" : 2048 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 341, "optime" : { "t" : 1358533259000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 341, "optime" : { "t" : 1358533259000, "i" : 1212 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 341, "optime" : { "t" : 1358533301000, "i" : 518 }, "optimeDate" : ISODate("2013-01-18T18:21:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 340, "optime" : { "t" : 1358533300000, "i" : 2903 }, "optimeDate" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:41Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 375, "optime" : { "t" : 1358533262000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:02Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 340, "optime" : { "t" : 1358533259000, "i" : 1212 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:41Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 340, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:40Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 340, "optime" : { "t" : 1358533300000, "i" : 2896 }, "optimeDate" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 340, "optime" : { "t" : 1358533301000, "i" : 153 }, "optimeDate" : ISODate("2013-01-18T18:21:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:41Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 340, "optime" : { "t" : 1358533259000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 374, "optime" : { "t" : 1358533259000, "i" : 1212 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 340, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:42Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 340, "optime" : { "t" : 1358533300000, "i" : 7793 }, "optimeDate" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:42Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 341, "optime" : { "t" : 1358533259000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 48 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 341, "optime" : { "t" : 1358533259000, "i" : 1212 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 28 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 373, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 341, "optime" : { "t" : 1358533300000, "i" : 47 }, "optimeDate" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 340, "optime" : { "t" : 1358533300000, "i" : 2736 }, "optimeDate" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:41Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 340, "optime" : { "t" : 1358533259000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 340, "optime" : { "t" : 1358533259000, "i" : 1212 }, "optimeDate" : ISODate("2013-01-18T18:20:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:41Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 340, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 373, "optime" : { "t" : 1358533302000, "i" : 1132 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31000| Fri Jan 18 13:21:46.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31000| Fri Jan 18 13:21:46.344 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:393649 196ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 380, "optime" : { "t" : 1358533307000, "i" : 4155 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 346, "optime" : { "t" : 1358533264000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 346, "optime" : { "t" : 1358533264000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 346, "optime" : { "t" : 1358533307000, "i" : 2868 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 345, "optime" : { "t" : 1358533306000, "i" : 1406 }, "optimeDate" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 380, "optime" : { "t" : 1358533264000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 345, "optime" : { "t" : 1358533264000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:47Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 345, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:46Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 345, "optime" : { "t" : 1358533306000, "i" : 1390 }, "optimeDate" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 345, "optime" : { "t" : 1358533307000, "i" : 169 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 345, "optime" : { "t" : 1358533264000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 379, "optime" : { "t" : 1358533264000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 345, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 345, "optime" : { "t" : 1358533307000, "i" : 156 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:47Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 346, "optime" : { "t" : 1358533264000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 36 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 346, "optime" : { "t" : 1358533264000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:47Z"), "pingMs" : 17 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 378, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 346, "optime" : { "t" : 1358533302000, "i" : 2048 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:46Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 345, "optime" : { "t" : 1358533306000, "i" : 1143 }, "optimeDate" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 345, "optime" : { "t" : 1358533264000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:46Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 345, "optime" : { "t" : 1358533264000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:47Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 345, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 378, "optime" : { "t" : 1358533307000, "i" : 4155 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31002| Fri Jan 18 13:21:50.335 [conn243] end connection 127.0.0.6:53589 (5 connections now open) m31002| Fri Jan 18 13:21:50.335 [initandlisten] connection accepted from 127.0.0.6:36268 #247 (6 connections now open) m31003| Fri Jan 18 13:21:50.358 [conn41] end connection 127.0.0.3:47129 (3 connections now open) m31003| Fri Jan 18 13:21:50.358 [initandlisten] connection accepted from 127.0.0.3:45694 #44 (5 connections now open) m31003| Fri Jan 18 13:21:50.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:21:51.025 [conn42] end connection 127.0.0.4:54411 (3 connections now open) m31003| Fri Jan 18 13:21:51.025 [initandlisten] connection accepted from 127.0.0.4:42410 #45 (4 connections now open) m31001| Fri Jan 18 13:21:51.379 [conn337] end connection 127.0.0.2:35462 (4 connections now open) m31001| Fri Jan 18 13:21:51.379 [initandlisten] connection accepted from 127.0.0.2:48592 #341 (5 connections now open) m31000| Fri Jan 18 13:21:52.335 [conn63] end connection 127.0.0.6:41941 (11 connections now open) m31000| Fri Jan 18 13:21:52.335 [initandlisten] connection accepted from 127.0.0.6:54067 #66 (12 connections now open) m31000| Fri Jan 18 13:21:52.358 [conn64] end connection 127.0.0.3:43533 (11 connections now open) m31000| Fri Jan 18 13:21:52.358 [initandlisten] connection accepted from 127.0.0.3:38999 #67 (12 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 385, "optime" : { "t" : 1358533312000, "i" : 7376 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 351, "optime" : { "t" : 1358533270000, "i" : 1542 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:52Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 351, "optime" : { "t" : 1358533270000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 351, "optime" : { "t" : 1358533311000, "i" : 3193 }, "optimeDate" : ISODate("2013-01-18T18:21:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 350, "optime" : { "t" : 1358533312000, "i" : 3004 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 385, "optime" : { "t" : 1358533270000, "i" : 1542 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 350, "optime" : { "t" : 1358533270000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 350, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 350, "optime" : { "t" : 1358533312000, "i" : 3004 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 350, "optime" : { "t" : 1358533311000, "i" : 46 }, "optimeDate" : ISODate("2013-01-18T18:21:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:51Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 350, "optime" : { "t" : 1358533270000, "i" : 1542 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:52Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 384, "optime" : { "t" : 1358533270000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 350, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 350, "optime" : { "t" : 1358533311000, "i" : 39 }, "optimeDate" : ISODate("2013-01-18T18:21:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:52Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 351, "optime" : { "t" : 1358533270000, "i" : 1542 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:52Z"), "pingMs" : 34 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 351, "optime" : { "t" : 1358533270000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 22 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 383, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 351, "optime" : { "t" : 1358533312000, "i" : 5565 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 350, "optime" : { "t" : 1358533312000, "i" : 2990 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 350, "optime" : { "t" : 1358533270000, "i" : 1542 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:52Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 350, "optime" : { "t" : 1358533270000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 350, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 383, "optime" : { "t" : 1358533312000, "i" : 7017 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31000| Fri Jan 18 13:21:53.026 [conn65] end connection 127.0.0.4:40949 (11 connections now open) m31000| Fri Jan 18 13:21:53.026 [initandlisten] connection accepted from 127.0.0.4:52311 #68 (12 connections now open) m31002| Fri Jan 18 13:21:54.181 [conn244] end connection 127.0.0.5:58593 (5 connections now open) m31002| Fri Jan 18 13:21:54.190 [initandlisten] connection accepted from 127.0.0.5:59718 #248 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 391, "optime" : { "t" : 1358533318000, "i" : 1417 }, "optimeDate" : ISODate("2013-01-18T18:21:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 357, "optime" : { "t" : 1358533275000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 357, "optime" : { "t" : 1358533275000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 357, "optime" : { "t" : 1358533317000, "i" : 2715 }, "optimeDate" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 356, "optime" : { "t" : 1358533316000, "i" : 3647 }, "optimeDate" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 391, "optime" : { "t" : 1358533275000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 356, "optime" : { "t" : 1358533275000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:57Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 356, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:57Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 356, "optime" : { "t" : 1358533316000, "i" : 3488 }, "optimeDate" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 356, "optime" : { "t" : 1358533316000, "i" : 9087 }, "optimeDate" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 356, "optime" : { "t" : 1358533275000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 390, "optime" : { "t" : 1358533275000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 356, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 356, "optime" : { "t" : 1358533316000, "i" : 8961 }, "optimeDate" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:58Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 357, "optime" : { "t" : 1358533275000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 27 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 357, "optime" : { "t" : 1358533275000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:57Z"), "pingMs" : 30 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 389, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 357, "optime" : { "t" : 1358533317000, "i" : 5099 }, "optimeDate" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:21:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 356, "optime" : { "t" : 1358533316000, "i" : 3441 }, "optimeDate" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 356, "optime" : { "t" : 1358533275000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:56Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 356, "optime" : { "t" : 1358533275000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:57Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 356, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:21:57Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 389, "optime" : { "t" : 1358533318000, "i" : 1417 }, "optimeDate" : ISODate("2013-01-18T18:21:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31000| Fri Jan 18 13:21:59.051 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:133125 nreturned:160000 reslen:2720020 133ms m31000| Fri Jan 18 13:21:59.149 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:131579 nreturned:160000 reslen:2720020 131ms m31003| Fri Jan 18 13:22:00.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Fri Jan 18 13:22:01.575 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:604982 302ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 396, "optime" : { "t" : 1358533323000, "i" : 3081 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 362, "optime" : { "t" : 1358533280000, "i" : 6319 }, "optimeDate" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:02Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 362, "optime" : { "t" : 1358533280000, "i" : 6319 }, "optimeDate" : ISODate("2013-01-18T18:21:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:03Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 362, "optime" : { "t" : 1358533321000, "i" : 562 }, "optimeDate" : ISODate("2013-01-18T18:22:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:02Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 361, "optime" : { "t" : 1358533322000, "i" : 2685 }, "optimeDate" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 396, "optime" : { "t" : 1358533281000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 361, "optime" : { "t" : 1358533281000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:03Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 361, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:03Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 361, "optime" : { "t" : 1358533322000, "i" : 2672 }, "optimeDate" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 361, "optime" : { "t" : 1358533323000, "i" : 346 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 361, "optime" : { "t" : 1358533281000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 395, "optime" : { "t" : 1358533281000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 361, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 361, "optime" : { "t" : 1358533323000, "i" : 327 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:03Z"), "myState" : 3, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 362, "optime" : { "t" : 1358533281000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 23 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 362, "optime" : { "t" : 1358533281000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:03Z"), "pingMs" : 29 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 394, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 362, "optime" : { "t" : 1358533322000, "i" : 5172 }, "optimeDate" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 361, "optime" : { "t" : 1358533322000, "i" : 2685 }, "optimeDate" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 361, "optime" : { "t" : 1358533281000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:02Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 361, "optime" : { "t" : 1358533281000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:03Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 361, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:02Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 394, "optime" : { "t" : 1358533323000, "i" : 3081 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ], "127.0.0.4:31002" : [ "127.0.0.5:31003" ] } } m31004| Fri Jan 18 13:22:03.380 [conn49] end connection 127.0.0.2:37988 (4 connections now open) m31004| Fri Jan 18 13:22:03.380 [initandlisten] connection accepted from 127.0.0.2:56896 #53 (5 connections now open) m31003| Fri Jan 18 13:22:04.337 [conn43] end connection 127.0.0.6:42211 (3 connections now open) m31003| Fri Jan 18 13:22:04.337 [initandlisten] connection accepted from 127.0.0.6:54558 #46 (4 connections now open) m31004| Fri Jan 18 13:22:04.360 [conn50] end connection 127.0.0.3:54858 (4 connections now open) m31004| Fri Jan 18 13:22:04.360 [initandlisten] connection accepted from 127.0.0.3:53179 #54 (5 connections now open) m31004| Fri Jan 18 13:22:05.027 [conn51] end connection 127.0.0.4:60626 (4 connections now open) m31004| Fri Jan 18 13:22:05.027 [initandlisten] connection accepted from 127.0.0.4:55886 #55 (5 connections now open) m31002| Fri Jan 18 13:22:05.381 [conn245] end connection 127.0.0.2:55487 (5 connections now open) m31002| Fri Jan 18 13:22:05.381 [initandlisten] connection accepted from 127.0.0.2:36251 #249 (6 connections now open) m31003| Fri Jan 18 13:22:05.888 [rsSync] replSet still syncing, not yet to minValid optime 50f991f5:10f5 m31001| Fri Jan 18 13:22:06.338 [conn338] end connection 127.0.0.6:60595 (4 connections now open) m31001| Fri Jan 18 13:22:06.338 [initandlisten] connection accepted from 127.0.0.6:55645 #342 (5 connections now open) m31002| Fri Jan 18 13:22:06.360 [conn246] end connection 127.0.0.3:38868 (5 connections now open) m31002| Fri Jan 18 13:22:06.361 [initandlisten] connection accepted from 127.0.0.3:53781 #250 (6 connections now open) m31000| Fri Jan 18 13:22:06.570 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:258449 129ms m31001| Fri Jan 18 13:22:07.028 [conn339] end connection 127.0.0.4:42642 (4 connections now open) m31001| Fri Jan 18 13:22:07.028 [initandlisten] connection accepted from 127.0.0.4:51928 #343 (5 connections now open) m31000| Fri Jan 18 13:22:08.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 401, "optime" : { "t" : 1358533328000, "i" : 5212 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 367, "optime" : { "t" : 1358533286000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 367, "optime" : { "t" : 1358533286000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 367, "optime" : { "t" : 1358533327000, "i" : 3481 }, "optimeDate" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 366, "optime" : { "t" : 1358533328000, "i" : 2940 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 401, "optime" : { "t" : 1358533286000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 366, "optime" : { "t" : 1358533286000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 366, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 366, "optime" : { "t" : 1358533328000, "i" : 2937 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 366, "optime" : { "t" : 1358533327000, "i" : 37 }, "optimeDate" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:07Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 366, "optime" : { "t" : 1358533285000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 400, "optime" : { "t" : 1358533286000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 366, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:08Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 366, "optime" : { "t" : 1358533326000, "i" : 5712 }, "optimeDate" : ISODate("2013-01-18T18:22:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:08Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 367, "optime" : { "t" : 1358533286000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:08Z"), "pingMs" : 14 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 367, "optime" : { "t" : 1358533286000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 23 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 399, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 367, "optime" : { "t" : 1358533327000, "i" : 5888 }, "optimeDate" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 366, "optime" : { "t" : 1358533328000, "i" : 2747 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 366, "optime" : { "t" : 1358533286000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 366, "optime" : { "t" : 1358533286000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 366, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 399, "optime" : { "t" : 1358533328000, "i" : 4925 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:22:10.141 [rsMgr] replSet I don't see a primary and I can't elect myself m31003| Fri Jan 18 13:22:10.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 475 more seconds m31003| Fri Jan 18 13:22:10.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 474 more seconds m31003| Fri Jan 18 13:22:10.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 474 more seconds m31003| Fri Jan 18 13:22:10.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 475 more seconds m31003| Fri Jan 18 13:22:10.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 474 more seconds m31003| Fri Jan 18 13:22:10.144 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 474 more seconds m31001| Fri Jan 18 13:22:11.194 [conn340] end connection 127.0.0.5:41284 (4 connections now open) m31001| Fri Jan 18 13:22:11.199 [initandlisten] connection accepted from 127.0.0.5:51003 #344 (5 connections now open) m31004| Fri Jan 18 13:22:12.641 [conn52] end connection 127.0.0.5:53328 (4 connections now open) m31004| Fri Jan 18 13:22:13.142 [initandlisten] connection accepted from 127.0.0.5:45001 #56 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 406, "optime" : { "t" : 1358533333000, "i" : 8561 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 372, "optime" : { "t" : 1358533291000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 372, "optime" : { "t" : 1358533291000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 372, "optime" : { "t" : 1358533333000, "i" : 3552 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 371, "optime" : { "t" : 1358533332000, "i" : 3105 }, "optimeDate" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 406, "optime" : { "t" : 1358533291000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 371, "optime" : { "t" : 1358533291000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:13Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 371, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:13Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 371, "optime" : { "t" : 1358533332000, "i" : 2866 }, "optimeDate" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 371, "optime" : { "t" : 1358533333000, "i" : 522 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 371, "optime" : { "t" : 1358533291000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 405, "optime" : { "t" : 1358533291000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 371, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 371, "optime" : { "t" : 1358533333000, "i" : 443 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:13Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 372, "optime" : { "t" : 1358533291000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 372, "optime" : { "t" : 1358533291000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:13Z"), "pingMs" : 14 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 404, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 372, "optime" : { "t" : 1358533333000, "i" : 1396 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 371, "optime" : { "t" : 1358533332000, "i" : 2885 }, "optimeDate" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 371, "optime" : { "t" : 1358533291000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:12Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 371, "optime" : { "t" : 1358533291000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:13Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 371, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:13Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 404, "optime" : { "t" : 1358533333000, "i" : 7889 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:22:15.145 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 470 more seconds m31003| Fri Jan 18 13:22:15.145 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 469 more seconds m31003| Fri Jan 18 13:22:15.145 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 469 more seconds m31003| Fri Jan 18 13:22:15.145 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 470 more seconds m31003| Fri Jan 18 13:22:15.145 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 469 more seconds m31003| Fri Jan 18 13:22:15.145 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 469 more seconds m31000| Fri Jan 18 13:22:18.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 412, "optime" : { "t" : 1358533339000, "i" : 2418 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 378, "optime" : { "t" : 1358533296000, "i" : 6167 }, "optimeDate" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:18Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 378, "optime" : { "t" : 1358533296000, "i" : 6167 }, "optimeDate" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:19Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 378, "optime" : { "t" : 1358533337000, "i" : 3308 }, "optimeDate" : ISODate("2013-01-18T18:22:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:18Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 377, "optime" : { "t" : 1358533338000, "i" : 3086 }, "optimeDate" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 412, "optime" : { "t" : 1358533297000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 377, "optime" : { "t" : 1358533297000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:19Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 377, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 377, "optime" : { "t" : 1358533338000, "i" : 3086 }, "optimeDate" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 377, "optime" : { "t" : 1358533339000, "i" : 192 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 377, "optime" : { "t" : 1358533297000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 411, "optime" : { "t" : 1358533297000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 377, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 377, "optime" : { "t" : 1358533339000, "i" : 75 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:19Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 378, "optime" : { "t" : 1358533296000, "i" : 6167 }, "optimeDate" : ISODate("2013-01-18T18:21:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:18Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 378, "optime" : { "t" : 1358533297000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:19Z"), "pingMs" : 18 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 410, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 378, "optime" : { "t" : 1358533338000, "i" : 843 }, "optimeDate" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 377, "optime" : { "t" : 1358533338000, "i" : 2886 }, "optimeDate" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 377, "optime" : { "t" : 1358533297000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:18Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 377, "optime" : { "t" : 1358533297000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:19Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 377, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 410, "optime" : { "t" : 1358533339000, "i" : 2418 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:22:20.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 465 more seconds m31003| Fri Jan 18 13:22:20.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 464 more seconds m31003| Fri Jan 18 13:22:20.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 464 more seconds m31003| Fri Jan 18 13:22:20.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 465 more seconds m31003| Fri Jan 18 13:22:20.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 464 more seconds m31003| Fri Jan 18 13:22:20.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 464 more seconds m31002| Fri Jan 18 13:22:20.340 [conn247] end connection 127.0.0.6:36268 (5 connections now open) m31002| Fri Jan 18 13:22:20.340 [initandlisten] connection accepted from 127.0.0.6:57837 #251 (7 connections now open) m31003| Fri Jan 18 13:22:20.362 [conn44] end connection 127.0.0.3:45694 (3 connections now open) m31003| Fri Jan 18 13:22:20.362 [initandlisten] connection accepted from 127.0.0.3:42757 #47 (4 connections now open) m31003| Fri Jan 18 13:22:21.030 [conn45] end connection 127.0.0.4:42410 (3 connections now open) m31003| Fri Jan 18 13:22:21.030 [initandlisten] connection accepted from 127.0.0.4:58758 #48 (4 connections now open) m31001| Fri Jan 18 13:22:21.383 [conn341] end connection 127.0.0.2:48592 (4 connections now open) m31001| Fri Jan 18 13:22:21.383 [initandlisten] connection accepted from 127.0.0.2:60007 #345 (5 connections now open) m31000| Fri Jan 18 13:22:22.340 [conn66] end connection 127.0.0.6:54067 (11 connections now open) m31000| Fri Jan 18 13:22:22.340 [initandlisten] connection accepted from 127.0.0.6:37660 #69 (12 connections now open) m31000| Fri Jan 18 13:22:22.362 [conn67] end connection 127.0.0.3:38999 (11 connections now open) m31000| Fri Jan 18 13:22:22.362 [initandlisten] connection accepted from 127.0.0.3:52244 #70 (12 connections now open) m31003| Fri Jan 18 13:22:22.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Fri Jan 18 13:22:23.030 [conn68] end connection 127.0.0.4:52311 (11 connections now open) m31000| Fri Jan 18 13:22:23.030 [initandlisten] connection accepted from 127.0.0.4:46378 #71 (12 connections now open) m31002| Fri Jan 18 13:22:24.602 [conn248] end connection 127.0.0.5:59718 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 417, "optime" : { "t" : 1358533344000, "i" : 5423 }, "optimeDate" : ISODate("2013-01-18T18:22:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 383, "optime" : { "t" : 1358533302000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:24Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 383, "optime" : { "t" : 1358533302000, "i" : 403 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:21:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 383, "optime" : { "t" : 1358533343000, "i" : 2741 }, "optimeDate" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:24Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 382, "optime" : { "t" : 1358533344000, "i" : 2774 }, "optimeDate" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 417, "optime" : { "t" : 1358533302000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 382, "optime" : { "t" : 1358533302000, "i" : 403 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 382, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 382, "optime" : { "t" : 1358533344000, "i" : 2771 }, "optimeDate" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 382, "optime" : { "t" : 1358533343000, "i" : 151 }, "optimeDate" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:23Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 382, "optime" : { "t" : 1358533302000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:24Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 416, "optime" : { "t" : 1358533302000, "i" : 403 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 382, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 382, "optime" : { "t" : 1358533343000, "i" : 151 }, "optimeDate" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:24Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:24Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 383, "optime" : { "t" : 1358533302000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:24Z"), "pingMs" : 33 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 383, "optime" : { "t" : 1358533302000, "i" : 403 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:23Z"), "pingMs" : 17 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 415, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 383, "optime" : { "t" : 1358533343000, "i" : 1066 }, "optimeDate" : ISODate("2013-01-18T18:22:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 382, "optime" : { "t" : 1358533344000, "i" : 2750 }, "optimeDate" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 382, "optime" : { "t" : 1358533302000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:24Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 382, "optime" : { "t" : 1358533302000, "i" : 403 }, "optimeDate" : ISODate("2013-01-18T18:21:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 382, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 415, "optime" : { "t" : 1358533344000, "i" : 5423 }, "optimeDate" : ISODate("2013-01-18T18:22:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:22:24.648 [initandlisten] connection accepted from 127.0.0.5:43391 #252 (6 connections now open) m31003| Fri Jan 18 13:22:25.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 460 more seconds m31003| Fri Jan 18 13:22:25.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 459 more seconds m31003| Fri Jan 18 13:22:25.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 459 more seconds m31003| Fri Jan 18 13:22:25.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 460 more seconds m31003| Fri Jan 18 13:22:25.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 459 more seconds m31003| Fri Jan 18 13:22:25.146 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 459 more seconds m31000| Fri Jan 18 13:22:28.318 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:343925 171ms m31000| Fri Jan 18 13:22:28.895 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:209392 104ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 422, "optime" : { "t" : 1358533349000, "i" : 6894 }, "optimeDate" : ISODate("2013-01-18T18:22:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 388, "optime" : { "t" : 1358533307000, "i" : 465 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 388, "optime" : { "t" : 1358533307000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 388, "optime" : { "t" : 1358533349000, "i" : 3545 }, "optimeDate" : ISODate("2013-01-18T18:22:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 387, "optime" : { "t" : 1358533348000, "i" : 2526 }, "optimeDate" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 422, "optime" : { "t" : 1358533307000, "i" : 465 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 387, "optime" : { "t" : 1358533307000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:29Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 387, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:29Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 387, "optime" : { "t" : 1358533348000, "i" : 2490 }, "optimeDate" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 387, "optime" : { "t" : 1358533348000, "i" : 10159 }, "optimeDate" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 387, "optime" : { "t" : 1358533307000, "i" : 465 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 421, "optime" : { "t" : 1358533307000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 387, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 387, "optime" : { "t" : 1358533348000, "i" : 10143 }, "optimeDate" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:29Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 388, "optime" : { "t" : 1358533307000, "i" : 465 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 43 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 388, "optime" : { "t" : 1358533307000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:29Z"), "pingMs" : 13 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 420, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 388, "optime" : { "t" : 1358533348000, "i" : 897 }, "optimeDate" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 387, "optime" : { "t" : 1358533348000, "i" : 1921 }, "optimeDate" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 387, "optime" : { "t" : 1358533307000, "i" : 465 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:28Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 387, "optime" : { "t" : 1358533307000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:29Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 387, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 420, "optime" : { "t" : 1358533349000, "i" : 6894 }, "optimeDate" : ISODate("2013-01-18T18:22:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:22:30.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 455 more seconds m31003| Fri Jan 18 13:22:30.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 454 more seconds m31003| Fri Jan 18 13:22:30.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 454 more seconds m31003| Fri Jan 18 13:22:30.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 455 more seconds m31003| Fri Jan 18 13:22:30.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 454 more seconds m31003| Fri Jan 18 13:22:30.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 454 more seconds m31003| Fri Jan 18 13:22:32.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31004| Fri Jan 18 13:22:33.385 [conn53] end connection 127.0.0.2:56896 (4 connections now open) m31004| Fri Jan 18 13:22:33.385 [initandlisten] connection accepted from 127.0.0.2:56452 #57 (5 connections now open) m31000| Fri Jan 18 13:22:33.581 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.3, filling with zeroes... m31003| Fri Jan 18 13:22:34.341 [conn46] end connection 127.0.0.6:54558 (3 connections now open) m31003| Fri Jan 18 13:22:34.342 [initandlisten] connection accepted from 127.0.0.6:36182 #49 (4 connections now open) m31004| Fri Jan 18 13:22:34.364 [conn54] end connection 127.0.0.3:53179 (4 connections now open) m31004| Fri Jan 18 13:22:34.364 [initandlisten] connection accepted from 127.0.0.3:38371 #58 (5 connections now open) m31000| Fri Jan 18 13:22:34.622 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.3, size: 128MB, took 1.04 secs m31000| Fri Jan 18 13:22:34.622 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:1041440 1041ms m31004| Fri Jan 18 13:22:34.623 [FileAllocator] allocating new datafile /data/db/testReplSet-4/foo.3, filling with zeroes... STATUS FROM : 127.0.0.2:31000 m31004| Fri Jan 18 13:22:35.032 [conn55] end connection 127.0.0.4:55886 (4 connections now open) m31004| Fri Jan 18 13:22:35.032 [initandlisten] connection accepted from 127.0.0.4:37009 #59 (5 connections now open) m31003| Fri Jan 18 13:22:35.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 450 more seconds m31003| Fri Jan 18 13:22:35.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 449 more seconds m31003| Fri Jan 18 13:22:35.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 449 more seconds m31003| Fri Jan 18 13:22:35.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 450 more seconds m31003| Fri Jan 18 13:22:35.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 449 more seconds m31003| Fri Jan 18 13:22:35.147 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 449 more seconds { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 428, "optime" : { "t" : 1358533355000, "i" : 5676 }, "optimeDate" : ISODate("2013-01-18T18:22:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 394, "optime" : { "t" : 1358533312000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:34Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 394, "optime" : { "t" : 1358533312000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:35Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 394, "optime" : { "t" : 1358533353000, "i" : 3362 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 393, "optime" : { "t" : 1358533353000, "i" : 5334 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 428, "optime" : { "t" : 1358533312000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 393, "optime" : { "t" : 1358533312000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:35Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 393, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 393, "optime" : { "t" : 1358533353000, "i" : 5334 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 393, "optime" : { "t" : 1358533354000, "i" : 7278 }, "optimeDate" : ISODate("2013-01-18T18:22:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 393, "optime" : { "t" : 1358533312000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 427, "optime" : { "t" : 1358533312000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 393, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 393, "optime" : { "t" : 1358533353000, "i" : 5334 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:35Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 394, "optime" : { "t" : 1358533312000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:34Z"), "pingMs" : 27 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 394, "optime" : { "t" : 1358533312000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:35Z"), "pingMs" : 16 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 426, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 394, "optime" : { "t" : 1358533353000, "i" : 427 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 393, "optime" : { "t" : 1358533353000, "i" : 5334 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 393, "optime" : { "t" : 1358533312000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:34Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 393, "optime" : { "t" : 1358533312000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:35Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 393, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 426, "optime" : { "t" : 1358533353000, "i" : 5334 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:22:35.385 [conn249] end connection 127.0.0.2:36251 (5 connections now open) m31002| Fri Jan 18 13:22:35.386 [initandlisten] connection accepted from 127.0.0.2:39567 #253 (6 connections now open) m31004| Fri Jan 18 13:22:35.699 [FileAllocator] done allocating datafile /data/db/testReplSet-4/foo.3, size: 128MB, took 1.075 secs m31001| Fri Jan 18 13:22:36.342 [conn342] end connection 127.0.0.6:55645 (4 connections now open) m31001| Fri Jan 18 13:22:36.342 [initandlisten] connection accepted from 127.0.0.6:58808 #346 (6 connections now open) m31002| Fri Jan 18 13:22:36.364 [conn250] end connection 127.0.0.3:53781 (5 connections now open) m31002| Fri Jan 18 13:22:36.364 [initandlisten] connection accepted from 127.0.0.3:54209 #254 (6 connections now open) m31001| Fri Jan 18 13:22:37.032 [conn343] end connection 127.0.0.4:51928 (4 connections now open) m31001| Fri Jan 18 13:22:37.032 [initandlisten] connection accepted from 127.0.0.4:46124 #347 (5 connections now open) m31003| Fri Jan 18 13:22:40.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 445 more seconds m31003| Fri Jan 18 13:22:40.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 444 more seconds m31003| Fri Jan 18 13:22:40.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 444 more seconds m31003| Fri Jan 18 13:22:40.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 445 more seconds m31003| Fri Jan 18 13:22:40.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 444 more seconds m31003| Fri Jan 18 13:22:40.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 444 more seconds m31000| Fri Jan 18 13:22:40.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 433, "optime" : { "t" : 1358533360000, "i" : 5866 }, "optimeDate" : ISODate("2013-01-18T18:22:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 399, "optime" : { "t" : 1358533318000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 399, "optime" : { "t" : 1358533317000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 399, "optime" : { "t" : 1358533359000, "i" : 3512 }, "optimeDate" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 398, "optime" : { "t" : 1358533360000, "i" : 3406 }, "optimeDate" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 433, "optime" : { "t" : 1358533318000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:58Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 398, "optime" : { "t" : 1358533317000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 398, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 398, "optime" : { "t" : 1358533360000, "i" : 3391 }, "optimeDate" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 398, "optime" : { "t" : 1358533359000, "i" : 524 }, "optimeDate" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 398, "optime" : { "t" : 1358533318000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 432, "optime" : { "t" : 1358533317000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:57Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 398, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 398, "optime" : { "t" : 1358533359000, "i" : 436 }, "optimeDate" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:40Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 399, "optime" : { "t" : 1358533318000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:40Z"), "pingMs" : 19 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 399, "optime" : { "t" : 1358533317000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:39Z"), "pingMs" : 24 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 431, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 399, "optime" : { "t" : 1358533360000, "i" : 5411 }, "optimeDate" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 398, "optime" : { "t" : 1358533360000, "i" : 3237 }, "optimeDate" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 398, "optime" : { "t" : 1358533318000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:21:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 398, "optime" : { "t" : 1358533317000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:21:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 398, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:40Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 431, "optime" : { "t" : 1358533360000, "i" : 5866 }, "optimeDate" : ISODate("2013-01-18T18:22:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:22:41.601 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:135306 nreturned:160000 reslen:2720020 135ms m31001| Fri Jan 18 13:22:41.688 [conn344] end connection 127.0.0.5:51003 (4 connections now open) m31001| Fri Jan 18 13:22:41.688 [initandlisten] connection accepted from 127.0.0.5:39437 #348 (6 connections now open) m31000| Fri Jan 18 13:22:41.945 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:137552 nreturned:160000 reslen:2720020 137ms m31003| Fri Jan 18 13:22:45.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 440 more seconds m31003| Fri Jan 18 13:22:45.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 439 more seconds m31003| Fri Jan 18 13:22:45.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 439 more seconds m31003| Fri Jan 18 13:22:45.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 440 more seconds m31003| Fri Jan 18 13:22:45.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 439 more seconds m31003| Fri Jan 18 13:22:45.148 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 439 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 439, "optime" : { "t" : 1358533366000, "i" : 705 }, "optimeDate" : ISODate("2013-01-18T18:22:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 405, "optime" : { "t" : 1358533323000, "i" : 1578 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 405, "optime" : { "t" : 1358533323000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 405, "optime" : { "t" : 1358533365000, "i" : 2710 }, "optimeDate" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 404, "optime" : { "t" : 1358533364000, "i" : 2964 }, "optimeDate" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 439, "optime" : { "t" : 1358533323000, "i" : 1578 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 404, "optime" : { "t" : 1358533323000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:45Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 404, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:45Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 404, "optime" : { "t" : 1358533364000, "i" : 2946 }, "optimeDate" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 404, "optime" : { "t" : 1358533365000, "i" : 155 }, "optimeDate" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 404, "optime" : { "t" : 1358533323000, "i" : 1578 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 438, "optime" : { "t" : 1358533323000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 404, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 404, "optime" : { "t" : 1358533365000, "i" : 155 }, "optimeDate" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:46Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 405, "optime" : { "t" : 1358533323000, "i" : 1578 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 19 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 405, "optime" : { "t" : 1358533323000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:45Z"), "pingMs" : 28 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 437, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 405, "optime" : { "t" : 1358533365000, "i" : 5291 }, "optimeDate" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 404, "optime" : { "t" : 1358533364000, "i" : 2803 }, "optimeDate" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 404, "optime" : { "t" : 1358533323000, "i" : 1578 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 404, "optime" : { "t" : 1358533323000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:45Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 404, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:45Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 437, "optime" : { "t" : 1358533366000, "i" : 2 }, "optimeDate" : ISODate("2013-01-18T18:22:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:22:50.149 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 435 more seconds m31003| Fri Jan 18 13:22:50.149 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 434 more seconds m31003| Fri Jan 18 13:22:50.149 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 434 more seconds m31003| Fri Jan 18 13:22:50.149 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 435 more seconds m31003| Fri Jan 18 13:22:50.149 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 434 more seconds m31003| Fri Jan 18 13:22:50.149 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 434 more seconds m31000| Fri Jan 18 13:22:50.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31002| Fri Jan 18 13:22:50.344 [conn251] end connection 127.0.0.6:57837 (5 connections now open) m31002| Fri Jan 18 13:22:50.344 [initandlisten] connection accepted from 127.0.0.6:58267 #255 (6 connections now open) m31003| Fri Jan 18 13:22:50.366 [conn47] end connection 127.0.0.3:42757 (3 connections now open) m31003| Fri Jan 18 13:22:50.366 [initandlisten] connection accepted from 127.0.0.3:49559 #50 (4 connections now open) m31004| Fri Jan 18 13:22:50.646 [conn56] end connection 127.0.0.5:45001 (4 connections now open) m31003| Fri Jan 18 13:22:51.034 [conn48] end connection 127.0.0.4:58758 (3 connections now open) m31003| Fri Jan 18 13:22:51.034 [initandlisten] connection accepted from 127.0.0.4:33250 #51 (5 connections now open) m31004| Fri Jan 18 13:22:51.146 [initandlisten] connection accepted from 127.0.0.5:37538 #60 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31001| Fri Jan 18 13:22:51.388 [conn345] end connection 127.0.0.2:60007 (4 connections now open) m31001| Fri Jan 18 13:22:51.388 [initandlisten] connection accepted from 127.0.0.2:33062 #349 (5 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 444, "optime" : { "t" : 1358533371000, "i" : 3062 }, "optimeDate" : ISODate("2013-01-18T18:22:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 410, "optime" : { "t" : 1358533328000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 410, "optime" : { "t" : 1358533328000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 410, "optime" : { "t" : 1358533371000, "i" : 830 }, "optimeDate" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 409, "optime" : { "t" : 1358533370000, "i" : 3213 }, "optimeDate" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:51Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 444, "optime" : { "t" : 1358533328000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 409, "optime" : { "t" : 1358533328000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:51Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 409, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 409, "optime" : { "t" : 1358533370000, "i" : 3163 }, "optimeDate" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 409, "optime" : { "t" : 1358533371000, "i" : 136 }, "optimeDate" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:51Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 409, "optime" : { "t" : 1358533328000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 443, "optime" : { "t" : 1358533328000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 409, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 409, "optime" : { "t" : 1358533371000, "i" : 135 }, "optimeDate" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:51Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 410, "optime" : { "t" : 1358533328000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:50Z"), "pingMs" : 12 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 410, "optime" : { "t" : 1358533328000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:51Z"), "pingMs" : 15 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 442, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 410, "optime" : { "t" : 1358533371000, "i" : 830 }, "optimeDate" : ISODate("2013-01-18T18:22:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 409, "optime" : { "t" : 1358533370000, "i" : 2758 }, "optimeDate" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:51Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 409, "optime" : { "t" : 1358533328000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:50Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 409, "optime" : { "t" : 1358533328000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:51Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 409, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:51Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 442, "optime" : { "t" : 1358533371000, "i" : 2319 }, "optimeDate" : ISODate("2013-01-18T18:22:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:22:52.344 [conn69] end connection 127.0.0.6:37660 (11 connections now open) m31000| Fri Jan 18 13:22:52.344 [initandlisten] connection accepted from 127.0.0.6:58534 #72 (13 connections now open) m31000| Fri Jan 18 13:22:52.367 [conn70] end connection 127.0.0.3:52244 (11 connections now open) m31000| Fri Jan 18 13:22:52.367 [initandlisten] connection accepted from 127.0.0.3:57535 #73 (13 connections now open) m31000| Fri Jan 18 13:22:53.034 [conn71] end connection 127.0.0.4:46378 (11 connections now open) m31000| Fri Jan 18 13:22:53.034 [initandlisten] connection accepted from 127.0.0.4:55050 #74 (13 connections now open) m31003| Fri Jan 18 13:22:54.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Fri Jan 18 13:22:54.954 [conn252] end connection 127.0.0.5:43391 (5 connections now open) m31002| Fri Jan 18 13:22:55.101 [initandlisten] connection accepted from 127.0.0.5:41580 #256 (6 connections now open) m31003| Fri Jan 18 13:22:55.149 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 430 more seconds m31003| Fri Jan 18 13:22:55.149 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 429 more seconds m31003| Fri Jan 18 13:22:55.149 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 429 more seconds m31003| Fri Jan 18 13:22:55.150 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 430 more seconds m31003| Fri Jan 18 13:22:55.150 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 429 more seconds m31003| Fri Jan 18 13:22:55.150 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 429 more seconds m31000| Fri Jan 18 13:22:56.522 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:660199 330ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 449, "optime" : { "t" : 1358533376000, "i" : 7061 }, "optimeDate" : ISODate("2013-01-18T18:22:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 415, "optime" : { "t" : 1358533333000, "i" : 6579 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:56Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 415, "optime" : { "t" : 1358533333000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 415, "optime" : { "t" : 1358533375000, "i" : 3488 }, "optimeDate" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:56Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 414, "optime" : { "t" : 1358533376000, "i" : 1701 }, "optimeDate" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 449, "optime" : { "t" : 1358533336000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:16Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 414, "optime" : { "t" : 1358533336000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 414, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 414, "optime" : { "t" : 1358533376000, "i" : 571 }, "optimeDate" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 414, "optime" : { "t" : 1358533375000, "i" : 201 }, "optimeDate" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 414, "optime" : { "t" : 1358533333000, "i" : 6579 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:56Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 448, "optime" : { "t" : 1358533336000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:16Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 414, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:55Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 414, "optime" : { "t" : 1358533375000, "i" : 201 }, "optimeDate" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:56Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:56Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 415, "optime" : { "t" : 1358533333000, "i" : 6579 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:56Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 415, "optime" : { "t" : 1358533333000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 27 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 447, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 415, "optime" : { "t" : 1358533376000, "i" : 571 }, "optimeDate" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:56Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:22:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 414, "optime" : { "t" : 1358533376000, "i" : 1701 }, "optimeDate" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 414, "optime" : { "t" : 1358533333000, "i" : 6579 }, "optimeDate" : ISODate("2013-01-18T18:22:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:22:56Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 414, "optime" : { "t" : 1358533336000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 414, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 447, "optime" : { "t" : 1358533376000, "i" : 7061 }, "optimeDate" : ISODate("2013-01-18T18:22:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:23:00.150 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 425 more seconds m31003| Fri Jan 18 13:23:00.150 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 424 more seconds m31003| Fri Jan 18 13:23:00.150 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 424 more seconds m31003| Fri Jan 18 13:23:00.150 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 425 more seconds m31003| Fri Jan 18 13:23:00.150 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 424 more seconds m31003| Fri Jan 18 13:23:00.150 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 424 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 455, "optime" : { "t" : 1358533382000, "i" : 2553 }, "optimeDate" : ISODate("2013-01-18T18:23:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 421, "optime" : { "t" : 1358533339000, "i" : 1322 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 421, "optime" : { "t" : 1358533339000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 421, "optime" : { "t" : 1358533381000, "i" : 3158 }, "optimeDate" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 420, "optime" : { "t" : 1358533380000, "i" : 2979 }, "optimeDate" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 455, "optime" : { "t" : 1358533342000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:22Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 420, "optime" : { "t" : 1358533339000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:01Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 420, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:01Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 420, "optime" : { "t" : 1358533380000, "i" : 2976 }, "optimeDate" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 420, "optime" : { "t" : 1358533381000, "i" : 258 }, "optimeDate" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 420, "optime" : { "t" : 1358533339000, "i" : 1322 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 454, "optime" : { "t" : 1358533339000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 420, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:01Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 420, "optime" : { "t" : 1358533380000, "i" : 8365 }, "optimeDate" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:02Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 421, "optime" : { "t" : 1358533339000, "i" : 1322 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 16 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 421, "optime" : { "t" : 1358533339000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 38 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 453, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 421, "optime" : { "t" : 1358533381000, "i" : 412 }, "optimeDate" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 420, "optime" : { "t" : 1358533380000, "i" : 2979 }, "optimeDate" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 420, "optime" : { "t" : 1358533339000, "i" : 1322 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 420, "optime" : { "t" : 1358533339000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:01Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 420, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:01Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 453, "optime" : { "t" : 1358533382000, "i" : 2182 }, "optimeDate" : ISODate("2013-01-18T18:23:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:23:03.390 [conn57] end connection 127.0.0.2:56452 (4 connections now open) m31004| Fri Jan 18 13:23:03.390 [initandlisten] connection accepted from 127.0.0.2:39883 #61 (5 connections now open) m31003| Fri Jan 18 13:23:04.346 [conn49] end connection 127.0.0.6:36182 (3 connections now open) m31003| Fri Jan 18 13:23:04.346 [initandlisten] connection accepted from 127.0.0.6:54719 #52 (4 connections now open) m31004| Fri Jan 18 13:23:04.368 [conn58] end connection 127.0.0.3:38371 (4 connections now open) m31004| Fri Jan 18 13:23:04.368 [initandlisten] connection accepted from 127.0.0.3:50920 #62 (6 connections now open) m31003| Fri Jan 18 13:23:04.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31004| Fri Jan 18 13:23:05.036 [conn59] end connection 127.0.0.4:37009 (4 connections now open) m31004| Fri Jan 18 13:23:05.036 [initandlisten] connection accepted from 127.0.0.4:38780 #63 (5 connections now open) m31003| Fri Jan 18 13:23:05.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 420 more seconds m31003| Fri Jan 18 13:23:05.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 419 more seconds m31003| Fri Jan 18 13:23:05.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 419 more seconds m31003| Fri Jan 18 13:23:05.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 420 more seconds m31003| Fri Jan 18 13:23:05.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 419 more seconds m31003| Fri Jan 18 13:23:05.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 419 more seconds m31002| Fri Jan 18 13:23:05.390 [conn253] end connection 127.0.0.2:39567 (5 connections now open) m31002| Fri Jan 18 13:23:05.390 [initandlisten] connection accepted from 127.0.0.2:46413 #257 (6 connections now open) m31003| Fri Jan 18 13:23:05.895 [rsSync] replSet still syncing, not yet to minValid optime 50f991f5:10f5 m31001| Fri Jan 18 13:23:06.346 [conn346] end connection 127.0.0.6:58808 (4 connections now open) m31001| Fri Jan 18 13:23:06.346 [initandlisten] connection accepted from 127.0.0.6:40483 #350 (5 connections now open) m31002| Fri Jan 18 13:23:06.368 [conn254] end connection 127.0.0.3:54209 (5 connections now open) m31002| Fri Jan 18 13:23:06.369 [initandlisten] connection accepted from 127.0.0.3:40868 #258 (6 connections now open) m31001| Fri Jan 18 13:23:07.036 [conn347] end connection 127.0.0.4:46124 (4 connections now open) m31001| Fri Jan 18 13:23:07.036 [initandlisten] connection accepted from 127.0.0.4:55663 #351 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:23:08.551 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:4919557 2459ms m31000| Fri Jan 18 13:23:08.553 [conn24] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185723707855858 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:96 nreturned:3 reslen:329 2461ms m31000| Fri Jan 18 13:23:08.553 [conn26] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185723782124859 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:133 nreturned:4 reslen:432 2461ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 462, "optime" : { "t" : 1358533389000, "i" : 4787 }, "optimeDate" : ISODate("2013-01-18T18:23:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 428, "optime" : { "t" : 1358533344000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 428, "optime" : { "t" : 1358533344000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:09Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 428, "optime" : { "t" : 1358533386000, "i" : 1045 }, "optimeDate" : ISODate("2013-01-18T18:23:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 427, "optime" : { "t" : 1358533386000, "i" : 1045 }, "optimeDate" : ISODate("2013-01-18T18:23:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 462, "optime" : { "t" : 1358533348000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:22:28Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 427, "optime" : { "t" : 1358533347000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:09Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 427, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 427, "optime" : { "t" : 1358533386000, "i" : 1045 }, "optimeDate" : ISODate("2013-01-18T18:23:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 427, "optime" : { "t" : 1358533388000, "i" : 7837 }, "optimeDate" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 427, "optime" : { "t" : 1358533348000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 461, "optime" : { "t" : 1358533349000, "i" : 4844 }, "optimeDate" : ISODate("2013-01-18T18:22:29Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 427, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 427, "optime" : { "t" : 1358533388000, "i" : 7837 }, "optimeDate" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:09Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 428, "optime" : { "t" : 1358533347000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:08Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 428, "optime" : { "t" : 1358533344000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:09Z"), "pingMs" : 34 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 460, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 428, "optime" : { "t" : 1358533388000, "i" : 2268 }, "optimeDate" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 427, "optime" : { "t" : 1358533386000, "i" : 1045 }, "optimeDate" : ISODate("2013-01-18T18:23:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 427, "optime" : { "t" : 1358533347000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 427, "optime" : { "t" : 1358533347000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:09Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 427, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:08Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 460, "optime" : { "t" : 1358533389000, "i" : 4787 }, "optimeDate" : ISODate("2013-01-18T18:23:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:23:10.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 415 more seconds m31003| Fri Jan 18 13:23:10.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 414 more seconds m31003| Fri Jan 18 13:23:10.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 414 more seconds m31003| Fri Jan 18 13:23:10.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 415 more seconds m31003| Fri Jan 18 13:23:10.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 414 more seconds m31003| Fri Jan 18 13:23:10.151 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 414 more seconds m31001| Fri Jan 18 13:23:12.127 [conn348] end connection 127.0.0.5:39437 (4 connections now open) m31001| Fri Jan 18 13:23:12.201 [initandlisten] connection accepted from 127.0.0.5:39979 #352 (5 connections now open) m31000| Fri Jan 18 13:23:12.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 467, "optime" : { "t" : 1358533394000, "i" : 4941 }, "optimeDate" : ISODate("2013-01-18T18:23:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 433, "optime" : { "t" : 1358533352000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:14Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 433, "optime" : { "t" : 1358533352000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 433, "optime" : { "t" : 1358533393000, "i" : 2084 }, "optimeDate" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:14Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 432, "optime" : { "t" : 1358533394000, "i" : 2952 }, "optimeDate" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 467, "optime" : { "t" : 1358533353000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 432, "optime" : { "t" : 1358533353000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 432, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 432, "optime" : { "t" : 1358533394000, "i" : 2919 }, "optimeDate" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 432, "optime" : { "t" : 1358533392000, "i" : 3983 }, "optimeDate" : ISODate("2013-01-18T18:23:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 432, "optime" : { "t" : 1358533352000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:14Z"), "pingMs" : 210 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 466, "optime" : { "t" : 1358533353000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 432, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:13Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 432, "optime" : { "t" : 1358533392000, "i" : 3973 }, "optimeDate" : ISODate("2013-01-18T18:23:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:14Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:14Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:22:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 433, "optime" : { "t" : 1358533353000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:14Z"), "pingMs" : 40 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 433, "optime" : { "t" : 1358533352000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 32 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 465, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 433, "optime" : { "t" : 1358533393000, "i" : 6205 }, "optimeDate" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 432, "optime" : { "t" : 1358533394000, "i" : 2731 }, "optimeDate" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 432, "optime" : { "t" : 1358533353000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:14Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 432, "optime" : { "t" : 1358533353000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 432, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 465, "optime" : { "t" : 1358533394000, "i" : 4941 }, "optimeDate" : ISODate("2013-01-18T18:23:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:23:14.690 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.3, filling with zeroes... m31001| Fri Jan 18 13:23:14.814 [FileAllocator] allocating new datafile /data/db/testReplSet-1/foo.3, filling with zeroes... m31003| Fri Jan 18 13:23:15.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 410 more seconds m31003| Fri Jan 18 13:23:15.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 409 more seconds m31003| Fri Jan 18 13:23:15.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 409 more seconds m31003| Fri Jan 18 13:23:15.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 410 more seconds m31003| Fri Jan 18 13:23:15.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 409 more seconds m31003| Fri Jan 18 13:23:15.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 409 more seconds m31002| Fri Jan 18 13:23:15.725 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.3, size: 128MB, took 1.034 secs m31001| Fri Jan 18 13:23:15.851 [FileAllocator] done allocating datafile /data/db/testReplSet-1/foo.3, size: 128MB, took 1.036 secs m31000| Fri Jan 18 13:23:18.276 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:710519 355ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 472, "optime" : { "t" : 1358533399000, "i" : 7028 }, "optimeDate" : ISODate("2013-01-18T18:23:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 438, "optime" : { "t" : 1358533358000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 438, "optime" : { "t" : 1358533358000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 438, "optime" : { "t" : 1358533399000, "i" : 3254 }, "optimeDate" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 437, "optime" : { "t" : 1358533398000, "i" : 1948 }, "optimeDate" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:19Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 472, "optime" : { "t" : 1358533359000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:39Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 437, "optime" : { "t" : 1358533358000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:19Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 437, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:18Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 437, "optime" : { "t" : 1358533398000, "i" : 1905 }, "optimeDate" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 437, "optime" : { "t" : 1358533399000, "i" : 248 }, "optimeDate" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:19Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 437, "optime" : { "t" : 1358533358000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 107 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 471, "optime" : { "t" : 1358533359000, "i" : 2493 }, "optimeDate" : ISODate("2013-01-18T18:22:39Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 437, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:19Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 437, "optime" : { "t" : 1358533399000, "i" : 239 }, "optimeDate" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:19Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 438, "optime" : { "t" : 1358533358000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 46 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 438, "optime" : { "t" : 1358533358000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 20 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 470, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 438, "optime" : { "t" : 1358533398000, "i" : 5650 }, "optimeDate" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 437, "optime" : { "t" : 1358533398000, "i" : 1475 }, "optimeDate" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:19Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 437, "optime" : { "t" : 1358533358000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:18Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 437, "optime" : { "t" : 1358533358000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:19Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 437, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:18Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 470, "optime" : { "t" : 1358533399000, "i" : 7028 }, "optimeDate" : ISODate("2013-01-18T18:23:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:23:20.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 405 more seconds m31003| Fri Jan 18 13:23:20.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 404 more seconds m31003| Fri Jan 18 13:23:20.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 404 more seconds m31003| Fri Jan 18 13:23:20.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 405 more seconds m31003| Fri Jan 18 13:23:20.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 404 more seconds m31003| Fri Jan 18 13:23:20.152 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 404 more seconds m31002| Fri Jan 18 13:23:20.348 [conn255] end connection 127.0.0.6:58267 (5 connections now open) m31002| Fri Jan 18 13:23:20.348 [initandlisten] connection accepted from 127.0.0.6:41571 #259 (6 connections now open) m31003| Fri Jan 18 13:23:20.370 [conn50] end connection 127.0.0.3:49559 (3 connections now open) m31003| Fri Jan 18 13:23:20.370 [initandlisten] connection accepted from 127.0.0.3:42916 #53 (4 connections now open) m31003| Fri Jan 18 13:23:21.038 [conn51] end connection 127.0.0.4:33250 (3 connections now open) m31003| Fri Jan 18 13:23:21.038 [initandlisten] connection accepted from 127.0.0.4:42723 #54 (4 connections now open) m31001| Fri Jan 18 13:23:21.392 [conn349] end connection 127.0.0.2:33062 (4 connections now open) m31001| Fri Jan 18 13:23:21.392 [initandlisten] connection accepted from 127.0.0.2:35501 #353 (5 connections now open) m31000| Fri Jan 18 13:23:22.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31000| Fri Jan 18 13:23:22.349 [conn72] end connection 127.0.0.6:58534 (11 connections now open) m31000| Fri Jan 18 13:23:22.349 [initandlisten] connection accepted from 127.0.0.6:39472 #75 (12 connections now open) m31000| Fri Jan 18 13:23:22.371 [conn73] end connection 127.0.0.3:57535 (11 connections now open) m31000| Fri Jan 18 13:23:22.371 [initandlisten] connection accepted from 127.0.0.3:40698 #76 (12 connections now open) m31000| Fri Jan 18 13:23:22.951 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:273655 136ms m31000| Fri Jan 18 13:23:23.038 [conn74] end connection 127.0.0.4:55050 (11 connections now open) m31000| Fri Jan 18 13:23:23.038 [initandlisten] connection accepted from 127.0.0.4:46999 #77 (13 connections now open) m31000| Fri Jan 18 13:23:23.231 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 numYields: 1119 locks(micros) r:259821 nreturned:143284 reslen:2435848 133ms m31000| Fri Jan 18 13:23:24.056 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:102245 102ms m31000| Fri Jan 18 13:23:24.091 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 numYields: 1181 locks(micros) r:269993 nreturned:151125 reslen:2569145 137ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 477, "optime" : { "t" : 1358533404000, "i" : 8876 }, "optimeDate" : ISODate("2013-01-18T18:23:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 443, "optime" : { "t" : 1358533360000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:24Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 443, "optime" : { "t" : 1358533363000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 443, "optime" : { "t" : 1358533403000, "i" : 4174 }, "optimeDate" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:24Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 442, "optime" : { "t" : 1358533404000, "i" : 4735 }, "optimeDate" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 477, "optime" : { "t" : 1358533363000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:43Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 442, "optime" : { "t" : 1358533364000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 442, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:24Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 442, "optime" : { "t" : 1358533404000, "i" : 4735 }, "optimeDate" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 442, "optime" : { "t" : 1358533403000, "i" : 570 }, "optimeDate" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:23Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 442, "optime" : { "t" : 1358533360000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:24Z"), "pingMs" : 68 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 476, "optime" : { "t" : 1358533364000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:44Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 442, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:23Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 442, "optime" : { "t" : 1358533402000, "i" : 1402 }, "optimeDate" : ISODate("2013-01-18T18:23:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:24Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:24Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 443, "optime" : { "t" : 1358533363000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 29 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 443, "optime" : { "t" : 1358533363000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 12 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 475, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 443, "optime" : { "t" : 1358533403000, "i" : 6969 }, "optimeDate" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 442, "optime" : { "t" : 1358533404000, "i" : 4715 }, "optimeDate" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 442, "optime" : { "t" : 1358533363000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:24Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 442, "optime" : { "t" : 1358533364000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 442, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 475, "optime" : { "t" : 1358533404000, "i" : 8737 }, "optimeDate" : ISODate("2013-01-18T18:23:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:23:25.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 400 more seconds m31003| Fri Jan 18 13:23:25.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 399 more seconds m31003| Fri Jan 18 13:23:25.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 399 more seconds m31003| Fri Jan 18 13:23:25.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 400 more seconds m31003| Fri Jan 18 13:23:25.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 399 more seconds m31003| Fri Jan 18 13:23:25.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 399 more seconds m31002| Fri Jan 18 13:23:25.489 [conn256] end connection 127.0.0.5:41580 (5 connections now open) m31002| Fri Jan 18 13:23:25.538 [initandlisten] connection accepted from 127.0.0.5:45995 #260 (6 connections now open) m31003| Fri Jan 18 13:23:26.150 [rsMgr] replSet I don't see a primary and I can't elect myself m31003| Fri Jan 18 13:23:26.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Fri Jan 18 13:23:28.076 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:435095 217ms m31004| Fri Jan 18 13:23:28.650 [conn60] end connection 127.0.0.5:37538 (4 connections now open) m31004| Fri Jan 18 13:23:29.150 [initandlisten] connection accepted from 127.0.0.5:59289 #64 (5 connections now open) m31000| Fri Jan 18 13:23:29.476 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:2591333 1295ms m31000| Fri Jan 18 13:23:29.476 [conn23] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185725031169493 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:61 nreturned:1 reslen:123 1297ms m31000| Fri Jan 18 13:23:29.477 [conn24] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185723707855858 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:38 nreturned:3 reslen:329 1298ms m31000| Fri Jan 18 13:23:29.477 [conn26] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185723782124859 ntoreturn:0 keyUpdates:0 locks(micros) r:29 nreturned:4 reslen:432 1298ms STATUS FROM : 127.0.0.2:31000 m31003| Fri Jan 18 13:23:30.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 395 more seconds m31003| Fri Jan 18 13:23:30.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 394 more seconds m31003| Fri Jan 18 13:23:30.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 394 more seconds m31003| Fri Jan 18 13:23:30.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 395 more seconds m31003| Fri Jan 18 13:23:30.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 394 more seconds m31003| Fri Jan 18 13:23:30.153 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 394 more seconds m31000| Fri Jan 18 13:23:30.792 [conn24] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185723707855858 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:73 nreturned:1 reslen:123 1214ms m31000| Fri Jan 18 13:23:30.792 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:2424067 1212ms m31000| Fri Jan 18 13:23:30.793 [conn23] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185725031169493 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:46 nreturned:1 reslen:123 1214ms m31000| Fri Jan 18 13:23:30.793 [conn26] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185723782124859 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:163 nreturned:1 reslen:123 1214ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 484, "optime" : { "t" : 1358533411000, "i" : 11892 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 450, "optime" : { "t" : 1358533369000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 450, "optime" : { "t" : 1358533369000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 450, "optime" : { "t" : 1358533411000, "i" : 2475 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 449, "optime" : { "t" : 1358533409000, "i" : 2610 }, "optimeDate" : ISODate("2013-01-18T18:23:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 484, "optime" : { "t" : 1358533370000, "i" : 6907 }, "optimeDate" : ISODate("2013-01-18T18:22:50Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 449, "optime" : { "t" : 1358533369000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:31Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 449, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:30Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 449, "optime" : { "t" : 1358533409000, "i" : 2525 }, "optimeDate" : ISODate("2013-01-18T18:23:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 449, "optime" : { "t" : 1358533410000, "i" : 2560 }, "optimeDate" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 449, "optime" : { "t" : 1358533369000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 27 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 483, "optime" : { "t" : 1358533370000, "i" : 6907 }, "optimeDate" : ISODate("2013-01-18T18:22:50Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 449, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:31Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 449, "optime" : { "t" : 1358533410000, "i" : 2533 }, "optimeDate" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:31Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 450, "optime" : { "t" : 1358533369000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 33 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 450, "optime" : { "t" : 1358533370000, "i" : 6907 }, "optimeDate" : ISODate("2013-01-18T18:22:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 25 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 482, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 450, "optime" : { "t" : 1358533411000, "i" : 7351 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 449, "optime" : { "t" : 1358533409000, "i" : 2610 }, "optimeDate" : ISODate("2013-01-18T18:23:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 449, "optime" : { "t" : 1358533369000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:30Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 449, "optime" : { "t" : 1358533369000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:31Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 449, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:31Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 482, "optime" : { "t" : 1358533411000, "i" : 11892 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:23:33.394 [conn61] end connection 127.0.0.2:39883 (4 connections now open) m31004| Fri Jan 18 13:23:33.394 [initandlisten] connection accepted from 127.0.0.2:36292 #65 (5 connections now open) m31003| Fri Jan 18 13:23:34.351 [conn52] end connection 127.0.0.6:54719 (3 connections now open) m31003| Fri Jan 18 13:23:34.351 [initandlisten] connection accepted from 127.0.0.6:51248 #55 (5 connections now open) m31004| Fri Jan 18 13:23:34.372 [conn62] end connection 127.0.0.3:50920 (4 connections now open) m31004| Fri Jan 18 13:23:34.373 [initandlisten] connection accepted from 127.0.0.3:34036 #66 (5 connections now open) m31004| Fri Jan 18 13:23:35.040 [conn63] end connection 127.0.0.4:38780 (4 connections now open) m31004| Fri Jan 18 13:23:35.040 [initandlisten] connection accepted from 127.0.0.4:55053 #67 (5 connections now open) m31003| Fri Jan 18 13:23:35.154 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 390 more seconds m31003| Fri Jan 18 13:23:35.154 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 389 more seconds m31003| Fri Jan 18 13:23:35.154 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 389 more seconds m31003| Fri Jan 18 13:23:35.154 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 390 more seconds m31003| Fri Jan 18 13:23:35.154 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 389 more seconds m31003| Fri Jan 18 13:23:35.154 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 389 more seconds m31002| Fri Jan 18 13:23:35.394 [conn257] end connection 127.0.0.2:46413 (5 connections now open) m31002| Fri Jan 18 13:23:35.394 [initandlisten] connection accepted from 127.0.0.2:34459 #261 (6 connections now open) m31001| Fri Jan 18 13:23:36.351 [conn350] end connection 127.0.0.6:40483 (4 connections now open) m31001| Fri Jan 18 13:23:36.351 [initandlisten] connection accepted from 127.0.0.6:39078 #354 (5 connections now open) m31002| Fri Jan 18 13:23:36.373 [conn258] end connection 127.0.0.3:40868 (5 connections now open) m31002| Fri Jan 18 13:23:36.373 [initandlisten] connection accepted from 127.0.0.3:45263 #262 (6 connections now open) m31003| Fri Jan 18 13:23:36.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 490, "optime" : { "t" : 1358533417000, "i" : 1395 }, "optimeDate" : ISODate("2013-01-18T18:23:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 456, "optime" : { "t" : 1358533374000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:36Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 456, "optime" : { "t" : 1358533374000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:37Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 456, "optime" : { "t" : 1358533415000, "i" : 2705 }, "optimeDate" : ISODate("2013-01-18T18:23:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:36Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 455, "optime" : { "t" : 1358533416000, "i" : 4468 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 490, "optime" : { "t" : 1358533376000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:56Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 455, "optime" : { "t" : 1358533375000, "i" : 6193 }, "optimeDate" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 455, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:36Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 455, "optime" : { "t" : 1358533416000, "i" : 4453 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 455, "optime" : { "t" : 1358533416000, "i" : 11398 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 455, "optime" : { "t" : 1358533374000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:36Z"), "pingMs" : 16 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 489, "optime" : { "t" : 1358533376000, "i" : 6105 }, "optimeDate" : ISODate("2013-01-18T18:22:56Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 455, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 455, "optime" : { "t" : 1358533416000, "i" : 11393 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:37Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 456, "optime" : { "t" : 1358533375000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 24 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 456, "optime" : { "t" : 1358533374000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:22:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:37Z"), "pingMs" : 16 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 488, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 456, "optime" : { "t" : 1358533416000, "i" : 8185 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 455, "optime" : { "t" : 1358533416000, "i" : 4468 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 455, "optime" : { "t" : 1358533375000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:36Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 455, "optime" : { "t" : 1358533375000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:22:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:37Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 455, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:36Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 488, "optime" : { "t" : 1358533417000, "i" : 1231 }, "optimeDate" : ISODate("2013-01-18T18:23:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31001| Fri Jan 18 13:23:37.250 [conn351] end connection 127.0.0.4:55663 (4 connections now open) m31001| Fri Jan 18 13:23:37.250 [initandlisten] connection accepted from 127.0.0.4:55656 #355 (5 connections now open) m31003| Fri Jan 18 13:23:40.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 385 more seconds m31003| Fri Jan 18 13:23:40.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 384 more seconds m31003| Fri Jan 18 13:23:40.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 384 more seconds m31003| Fri Jan 18 13:23:40.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 385 more seconds m31003| Fri Jan 18 13:23:40.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 384 more seconds m31003| Fri Jan 18 13:23:40.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 384 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 495, "optime" : { "t" : 1358533422000, "i" : 5593 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 461, "optime" : { "t" : 1358533380000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:42Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 461, "optime" : { "t" : 1358533380000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 461, "optime" : { "t" : 1358533421000, "i" : 3244 }, "optimeDate" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:42Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 460, "optime" : { "t" : 1358533422000, "i" : 4411 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 495, "optime" : { "t" : 1358533381000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:01Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 460, "optime" : { "t" : 1358533382000, "i" : 2086 }, "optimeDate" : ISODate("2013-01-18T18:23:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 460, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 460, "optime" : { "t" : 1358533422000, "i" : 4368 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 460, "optime" : { "t" : 1358533421000, "i" : 544 }, "optimeDate" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:41Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 460, "optime" : { "t" : 1358533380000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:42Z"), "pingMs" : 7 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 494, "optime" : { "t" : 1358533382000, "i" : 2086 }, "optimeDate" : ISODate("2013-01-18T18:23:02Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 460, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:41Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 460, "optime" : { "t" : 1358533420000, "i" : 6417 }, "optimeDate" : ISODate("2013-01-18T18:23:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:42Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:42Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 461, "optime" : { "t" : 1358533380000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:42Z"), "pingMs" : 19 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 461, "optime" : { "t" : 1358533380000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 31 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 493, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 461, "optime" : { "t" : 1358533421000, "i" : 5539 }, "optimeDate" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 460, "optime" : { "t" : 1358533422000, "i" : 4211 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 460, "optime" : { "t" : 1358533381000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:42Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 460, "optime" : { "t" : 1358533382000, "i" : 2086 }, "optimeDate" : ISODate("2013-01-18T18:23:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 460, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 493, "optime" : { "t" : 1358533422000, "i" : 5593 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31001| Fri Jan 18 13:23:42.591 [conn352] end connection 127.0.0.5:39979 (4 connections now open) m31001| Fri Jan 18 13:23:42.591 [initandlisten] connection accepted from 127.0.0.5:34669 #356 (5 connections now open) m31000| Fri Jan 18 13:23:44.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:23:45.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 380 more seconds m31003| Fri Jan 18 13:23:45.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 379 more seconds m31003| Fri Jan 18 13:23:45.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 379 more seconds m31003| Fri Jan 18 13:23:45.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 380 more seconds m31003| Fri Jan 18 13:23:45.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 379 more seconds m31003| Fri Jan 18 13:23:45.155 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 379 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 500, "optime" : { "t" : 1358533427000, "i" : 6496 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 466, "optime" : { "t" : 1358533385000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 466, "optime" : { "t" : 1358533386000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 466, "optime" : { "t" : 1358533427000, "i" : 3030 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 465, "optime" : { "t" : 1358533426000, "i" : 2968 }, "optimeDate" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 500, "optime" : { "t" : 1358533385000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 465, "optime" : { "t" : 1358533385000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:47Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 465, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:46Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 465, "optime" : { "t" : 1358533426000, "i" : 2968 }, "optimeDate" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 465, "optime" : { "t" : 1358533427000, "i" : 85 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 465, "optime" : { "t" : 1358533385000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 3 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 499, "optime" : { "t" : 1358533386000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:06Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 465, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 465, "optime" : { "t" : 1358533427000, "i" : 53 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:47Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 466, "optime" : { "t" : 1358533385000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 25 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 466, "optime" : { "t" : 1358533385000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:47Z"), "pingMs" : 29 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 498, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 466, "optime" : { "t" : 1358533426000, "i" : 5777 }, "optimeDate" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 465, "optime" : { "t" : 1358533426000, "i" : 2968 }, "optimeDate" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 465, "optime" : { "t" : 1358533385000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:46Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 465, "optime" : { "t" : 1358533385000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:47Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 465, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:46Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 498, "optime" : { "t" : 1358533427000, "i" : 6496 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:23:48.332 [slaveTracking] update local.slaves query: { _id: ObjectId('50f9915cf8ae107b7569538b'), host: "127.0.0.3", ns: "local.oplog.rs" } update: { $set: { syncedTo: Timestamp 1358533360000|5866 } } nscanned:1 nupdated:1 fastmod:1 keyUpdates:0 locks(micros) w:158 425ms m31003| Fri Jan 18 13:23:50.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 375 more seconds m31003| Fri Jan 18 13:23:50.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 374 more seconds m31003| Fri Jan 18 13:23:50.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 374 more seconds m31003| Fri Jan 18 13:23:50.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 375 more seconds m31003| Fri Jan 18 13:23:50.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 374 more seconds m31003| Fri Jan 18 13:23:50.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 374 more seconds m31002| Fri Jan 18 13:23:50.353 [conn259] end connection 127.0.0.6:41571 (5 connections now open) m31002| Fri Jan 18 13:23:50.353 [initandlisten] connection accepted from 127.0.0.6:56790 #263 (6 connections now open) m31003| Fri Jan 18 13:23:50.375 [conn53] end connection 127.0.0.3:42916 (3 connections now open) m31003| Fri Jan 18 13:23:50.375 [initandlisten] connection accepted from 127.0.0.3:33802 #56 (4 connections now open) m31003| Fri Jan 18 13:23:51.043 [conn54] end connection 127.0.0.4:42723 (3 connections now open) m31003| Fri Jan 18 13:23:51.043 [initandlisten] connection accepted from 127.0.0.4:56388 #57 (5 connections now open) m31000| Fri Jan 18 13:23:51.334 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:352869 176ms m31001| Fri Jan 18 13:23:51.397 [conn353] end connection 127.0.0.2:35501 (4 connections now open) m31001| Fri Jan 18 13:23:51.397 [initandlisten] connection accepted from 127.0.0.2:59501 #357 (5 connections now open) m31000| Fri Jan 18 13:23:52.353 [conn75] end connection 127.0.0.6:39472 (11 connections now open) m31000| Fri Jan 18 13:23:52.353 [initandlisten] connection accepted from 127.0.0.6:37794 #78 (12 connections now open) m31000| Fri Jan 18 13:23:52.375 [conn76] end connection 127.0.0.3:40698 (11 connections now open) m31000| Fri Jan 18 13:23:52.375 [initandlisten] connection accepted from 127.0.0.3:39589 #79 (12 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:23:53.043 [conn77] end connection 127.0.0.4:46999 (11 connections now open) m31000| Fri Jan 18 13:23:53.043 [initandlisten] connection accepted from 127.0.0.4:45965 #80 (12 connections now open) m31000| Fri Jan 18 13:23:53.218 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:674434 337ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 506, "optime" : { "t" : 1358533433000, "i" : 458 }, "optimeDate" : ISODate("2013-01-18T18:23:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 472, "optime" : { "t" : 1358533389000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:52Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 472, "optime" : { "t" : 1358533389000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:53Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 472, "optime" : { "t" : 1358533431000, "i" : 2159 }, "optimeDate" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 471, "optime" : { "t" : 1358533432000, "i" : 3514 }, "optimeDate" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 506, "optime" : { "t" : 1358533392000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:12Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 471, "optime" : { "t" : 1358533389000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 471, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 471, "optime" : { "t" : 1358533432000, "i" : 3424 }, "optimeDate" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 471, "optime" : { "t" : 1358533432000, "i" : 7673 }, "optimeDate" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 471, "optime" : { "t" : 1358533389000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:52Z"), "pingMs" : 1 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 505, "optime" : { "t" : 1358533392000, "i" : 849 }, "optimeDate" : ISODate("2013-01-18T18:23:12Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 471, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 471, "optime" : { "t" : 1358533432000, "i" : 7171 }, "optimeDate" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:53Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 472, "optime" : { "t" : 1358533392000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 24 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 472, "optime" : { "t" : 1358533389000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:53Z"), "pingMs" : 14 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 504, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 472, "optime" : { "t" : 1358533431000, "i" : 6281 }, "optimeDate" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 471, "optime" : { "t" : 1358533432000, "i" : 3081 }, "optimeDate" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 471, "optime" : { "t" : 1358533389000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:52Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 471, "optime" : { "t" : 1358533389000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:53Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 471, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 504, "optime" : { "t" : 1358533433000, "i" : 458 }, "optimeDate" : ISODate("2013-01-18T18:23:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:23:54.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:23:55.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 370 more seconds m31003| Fri Jan 18 13:23:55.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 369 more seconds m31003| Fri Jan 18 13:23:55.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 369 more seconds m31003| Fri Jan 18 13:23:55.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 370 more seconds m31003| Fri Jan 18 13:23:55.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 369 more seconds m31003| Fri Jan 18 13:23:55.156 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 369 more seconds m31002| Fri Jan 18 13:23:55.898 [conn260] end connection 127.0.0.5:45995 (5 connections now open) m31002| Fri Jan 18 13:23:55.898 [initandlisten] connection accepted from 127.0.0.5:57259 #264 (7 connections now open) m31000| Fri Jan 18 13:23:58.003 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:352943 176ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 511, "optime" : { "t" : 1358533438000, "i" : 5863 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 477, "optime" : { "t" : 1358533397000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:58Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 477, "optime" : { "t" : 1358533394000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 477, "optime" : { "t" : 1358533437000, "i" : 4002 }, "optimeDate" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:58Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 476, "optime" : { "t" : 1358533438000, "i" : 5834 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 511, "optime" : { "t" : 1358533398000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:18Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 476, "optime" : { "t" : 1358533397000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 476, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 476, "optime" : { "t" : 1358533438000, "i" : 5443 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 476, "optime" : { "t" : 1358533437000, "i" : 881 }, "optimeDate" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 476, "optime" : { "t" : 1358533394000, "i" : 3928 }, "optimeDate" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:58Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 510, "optime" : { "t" : 1358533397000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:17Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 476, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:57Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 476, "optime" : { "t" : 1358533436000, "i" : 6389 }, "optimeDate" : ISODate("2013-01-18T18:23:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:58Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:58Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 477, "optime" : { "t" : 1358533394000, "i" : 3928 }, "optimeDate" : ISODate("2013-01-18T18:23:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:58Z"), "pingMs" : 43 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 477, "optime" : { "t" : 1358533397000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 12 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 509, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 477, "optime" : { "t" : 1358533436000, "i" : 3185 }, "optimeDate" : ISODate("2013-01-18T18:23:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:23:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 476, "optime" : { "t" : 1358533438000, "i" : 5398 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 476, "optime" : { "t" : 1358533398000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:23:58Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 476, "optime" : { "t" : 1358533397000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 476, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 509, "optime" : { "t" : 1358533438000, "i" : 5863 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:23:58.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:24:00.157 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 365 more seconds m31003| Fri Jan 18 13:24:00.157 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 364 more seconds m31003| Fri Jan 18 13:24:00.157 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 364 more seconds m31003| Fri Jan 18 13:24:00.157 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 365 more seconds m31003| Fri Jan 18 13:24:00.157 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 364 more seconds m31003| Fri Jan 18 13:24:00.157 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 364 more seconds m31004| Fri Jan 18 13:24:03.399 [conn65] end connection 127.0.0.2:36292 (4 connections now open) m31004| Fri Jan 18 13:24:03.399 [initandlisten] connection accepted from 127.0.0.2:41469 #68 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 516, "optime" : { "t" : 1358533443000, "i" : 4313 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 482, "optime" : { "t" : 1358533402000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 482, "optime" : { "t" : 1358533403000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 482, "optime" : { "t" : 1358533443000, "i" : 3747 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 481, "optime" : { "t" : 1358533442000, "i" : 2774 }, "optimeDate" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:03Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 516, "optime" : { "t" : 1358533402000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:22Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 481, "optime" : { "t" : 1358533402000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:03Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 481, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:02Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 481, "optime" : { "t" : 1358533442000, "i" : 2774 }, "optimeDate" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 481, "optime" : { "t" : 1358533443000, "i" : 492 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:03Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 481, "optime" : { "t" : 1358533402000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 515, "optime" : { "t" : 1358533403000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:23Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 481, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 481, "optime" : { "t" : 1358533443000, "i" : 75 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:03Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 482, "optime" : { "t" : 1358533402000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 30 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 482, "optime" : { "t" : 1358533399000, "i" : 4946 }, "optimeDate" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:03Z"), "pingMs" : 17 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 514, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 482, "optime" : { "t" : 1358533441000, "i" : 2120 }, "optimeDate" : ISODate("2013-01-18T18:24:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 481, "optime" : { "t" : 1358533442000, "i" : 2774 }, "optimeDate" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:03Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 481, "optime" : { "t" : 1358533399000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:02Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 481, "optime" : { "t" : 1358533402000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:03Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 481, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 514, "optime" : { "t" : 1358533443000, "i" : 4226 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:24:03.681 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:136675 nreturned:156716 reslen:2664192 136ms m31003| Fri Jan 18 13:24:04.355 [conn55] end connection 127.0.0.6:51248 (3 connections now open) m31003| Fri Jan 18 13:24:04.355 [initandlisten] connection accepted from 127.0.0.6:48635 #58 (5 connections now open) m31004| Fri Jan 18 13:24:04.377 [conn66] end connection 127.0.0.3:34036 (4 connections now open) m31004| Fri Jan 18 13:24:04.377 [initandlisten] connection accepted from 127.0.0.3:53124 #69 (6 connections now open) m31000| Fri Jan 18 13:24:04.519 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 numYields: 268 locks(micros) r:217512 nreturned:148875 reslen:2530895 136ms m31004| Fri Jan 18 13:24:05.045 [conn67] end connection 127.0.0.4:55053 (4 connections now open) m31004| Fri Jan 18 13:24:05.045 [initandlisten] connection accepted from 127.0.0.4:55506 #70 (5 connections now open) m31003| Fri Jan 18 13:24:05.159 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 360 more seconds m31003| Fri Jan 18 13:24:05.159 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 359 more seconds m31003| Fri Jan 18 13:24:05.159 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 359 more seconds m31003| Fri Jan 18 13:24:05.159 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 360 more seconds m31003| Fri Jan 18 13:24:05.159 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 359 more seconds m31003| Fri Jan 18 13:24:05.159 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 359 more seconds m31002| Fri Jan 18 13:24:05.399 [conn261] end connection 127.0.0.2:34459 (5 connections now open) m31002| Fri Jan 18 13:24:05.399 [initandlisten] connection accepted from 127.0.0.2:45699 #265 (6 connections now open) m31003| Fri Jan 18 13:24:05.903 [rsSync] replSet still syncing, not yet to minValid optime 50f991f5:10f5 m31001| Fri Jan 18 13:24:06.356 [conn354] end connection 127.0.0.6:39078 (4 connections now open) m31001| Fri Jan 18 13:24:06.356 [initandlisten] connection accepted from 127.0.0.6:59535 #358 (6 connections now open) m31002| Fri Jan 18 13:24:06.377 [conn262] end connection 127.0.0.3:45263 (5 connections now open) m31002| Fri Jan 18 13:24:06.377 [initandlisten] connection accepted from 127.0.0.3:57754 #266 (6 connections now open) m31004| Fri Jan 18 13:24:06.655 [conn64] end connection 127.0.0.5:59289 (4 connections now open) m31004| Fri Jan 18 13:24:07.155 [initandlisten] connection accepted from 127.0.0.5:46083 #71 (5 connections now open) m31001| Fri Jan 18 13:24:07.255 [conn355] end connection 127.0.0.4:55656 (4 connections now open) m31001| Fri Jan 18 13:24:07.255 [initandlisten] connection accepted from 127.0.0.4:58658 #359 (5 connections now open) m31000| Fri Jan 18 13:24:08.057 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:443967 221ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 521, "optime" : { "t" : 1358533448000, "i" : 7296 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 487, "optime" : { "t" : 1358533404000, "i" : 5649 }, "optimeDate" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 487, "optime" : { "t" : 1358533404000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 487, "optime" : { "t" : 1358533447000, "i" : 2402 }, "optimeDate" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 486, "optime" : { "t" : 1358533448000, "i" : 5034 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 521, "optime" : { "t" : 1358533407000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:27Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 486, "optime" : { "t" : 1358533407000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 486, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 486, "optime" : { "t" : 1358533448000, "i" : 5014 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 486, "optime" : { "t" : 1358533447000, "i" : 687 }, "optimeDate" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:07Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 486, "optime" : { "t" : 1358533404000, "i" : 5649 }, "optimeDate" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 520, "optime" : { "t" : 1358533407000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:27Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 486, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:08Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 486, "optime" : { "t" : 1358533447000, "i" : 423 }, "optimeDate" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:08Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 487, "optime" : { "t" : 1358533404000, "i" : 5649 }, "optimeDate" : ISODate("2013-01-18T18:23:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:08Z"), "pingMs" : 36 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 487, "optime" : { "t" : 1358533407000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 18 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 519, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 487, "optime" : { "t" : 1358533447000, "i" : 1426 }, "optimeDate" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 486, "optime" : { "t" : 1358533448000, "i" : 4969 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 486, "optime" : { "t" : 1358533407000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 486, "optime" : { "t" : 1358533407000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 486, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 519, "optime" : { "t" : 1358533448000, "i" : 7032 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:24:08.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:24:10.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 355 more seconds m31003| Fri Jan 18 13:24:10.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 354 more seconds m31003| Fri Jan 18 13:24:10.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 354 more seconds m31003| Fri Jan 18 13:24:10.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 355 more seconds m31003| Fri Jan 18 13:24:10.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 354 more seconds m31003| Fri Jan 18 13:24:10.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 354 more seconds m31001| Fri Jan 18 13:24:13.071 [conn356] end connection 127.0.0.5:34669 (4 connections now open) m31001| Fri Jan 18 13:24:13.148 [initandlisten] connection accepted from 127.0.0.5:35463 #360 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 526, "optime" : { "t" : 1358533453000, "i" : 8062 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 492, "optime" : { "t" : 1358533411000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 492, "optime" : { "t" : 1358533411000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 492, "optime" : { "t" : 1358533453000, "i" : 3407 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 491, "optime" : { "t" : 1358533452000, "i" : 3421 }, "optimeDate" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 526, "optime" : { "t" : 1358533411000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 491, "optime" : { "t" : 1358533411000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:13Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 491, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:13Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 491, "optime" : { "t" : 1358533452000, "i" : 3116 }, "optimeDate" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 491, "optime" : { "t" : 1358533453000, "i" : 178 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 491, "optime" : { "t" : 1358533411000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 525, "optime" : { "t" : 1358533411000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 491, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 491, "optime" : { "t" : 1358533453000, "i" : 151 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:13Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:23:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 492, "optime" : { "t" : 1358533411000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 17 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 492, "optime" : { "t" : 1358533411000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:13Z"), "pingMs" : 11 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 524, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 492, "optime" : { "t" : 1358533452000, "i" : 1306 }, "optimeDate" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 491, "optime" : { "t" : 1358533452000, "i" : 3266 }, "optimeDate" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 491, "optime" : { "t" : 1358533411000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:12Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 491, "optime" : { "t" : 1358533411000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:13Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 491, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 524, "optime" : { "t" : 1358533453000, "i" : 7478 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:24:15.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 350 more seconds m31003| Fri Jan 18 13:24:15.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 349 more seconds m31003| Fri Jan 18 13:24:15.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 349 more seconds m31003| Fri Jan 18 13:24:15.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 350 more seconds m31003| Fri Jan 18 13:24:15.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 349 more seconds m31003| Fri Jan 18 13:24:15.160 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 349 more seconds m31000| Fri Jan 18 13:24:16.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31000| Fri Jan 18 13:24:18.262 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:736440 368ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 532, "optime" : { "t" : 1358533459000, "i" : 531 }, "optimeDate" : ISODate("2013-01-18T18:24:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 498, "optime" : { "t" : 1358533416000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:18Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 498, "optime" : { "t" : 1358533416000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:19Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 498, "optime" : { "t" : 1358533457000, "i" : 4457 }, "optimeDate" : ISODate("2013-01-18T18:24:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:18Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 497, "optime" : { "t" : 1358533458000, "i" : 2128 }, "optimeDate" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 532, "optime" : { "t" : 1358533417000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:37Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 497, "optime" : { "t" : 1358533416000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 497, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 497, "optime" : { "t" : 1358533458000, "i" : 2039 }, "optimeDate" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 497, "optime" : { "t" : 1358533459000, "i" : 303 }, "optimeDate" : ISODate("2013-01-18T18:24:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 497, "optime" : { "t" : 1358533416000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:18Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 531, "optime" : { "t" : 1358533416000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 497, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 497, "optime" : { "t" : 1358533459000, "i" : 153 }, "optimeDate" : ISODate("2013-01-18T18:24:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:19Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 498, "optime" : { "t" : 1358533416000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:18Z"), "pingMs" : 19 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 498, "optime" : { "t" : 1358533416000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:19Z"), "pingMs" : 24 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 530, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 498, "optime" : { "t" : 1358533457000, "i" : 1369 }, "optimeDate" : ISODate("2013-01-18T18:24:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 497, "optime" : { "t" : 1358533458000, "i" : 1921 }, "optimeDate" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 497, "optime" : { "t" : 1358533417000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:18Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 497, "optime" : { "t" : 1358533416000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:19Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 497, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 530, "optime" : { "t" : 1358533459000, "i" : 531 }, "optimeDate" : ISODate("2013-01-18T18:24:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:24:20.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 345 more seconds m31003| Fri Jan 18 13:24:20.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 344 more seconds m31003| Fri Jan 18 13:24:20.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 344 more seconds m31003| Fri Jan 18 13:24:20.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 345 more seconds m31003| Fri Jan 18 13:24:20.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 344 more seconds m31003| Fri Jan 18 13:24:20.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 344 more seconds m31002| Fri Jan 18 13:24:20.358 [conn263] end connection 127.0.0.6:56790 (5 connections now open) m31002| Fri Jan 18 13:24:20.358 [initandlisten] connection accepted from 127.0.0.6:44252 #267 (6 connections now open) m31003| Fri Jan 18 13:24:20.379 [conn56] end connection 127.0.0.3:33802 (3 connections now open) m31003| Fri Jan 18 13:24:20.379 [initandlisten] connection accepted from 127.0.0.3:49280 #59 (4 connections now open) m31003| Fri Jan 18 13:24:21.047 [initandlisten] connection accepted from 127.0.0.4:37028 #60 (5 connections now open) m31003| Fri Jan 18 13:24:21.047 [conn57] end connection 127.0.0.4:56388 (3 connections now open) m31001| Fri Jan 18 13:24:21.402 [conn357] end connection 127.0.0.2:59501 (4 connections now open) m31001| Fri Jan 18 13:24:21.402 [initandlisten] connection accepted from 127.0.0.2:52064 #361 (5 connections now open) m31000| Fri Jan 18 13:24:22.358 [conn78] end connection 127.0.0.6:37794 (11 connections now open) m31000| Fri Jan 18 13:24:22.358 [initandlisten] connection accepted from 127.0.0.6:49398 #81 (12 connections now open) m31000| Fri Jan 18 13:24:22.379 [conn79] end connection 127.0.0.3:39589 (11 connections now open) m31000| Fri Jan 18 13:24:22.379 [initandlisten] connection accepted from 127.0.0.3:56992 #82 (12 connections now open) m31000| Fri Jan 18 13:24:23.047 [conn80] end connection 127.0.0.4:45965 (11 connections now open) m31000| Fri Jan 18 13:24:23.047 [initandlisten] connection accepted from 127.0.0.4:52611 #83 (12 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 537, "optime" : { "t" : 1358533464000, "i" : 4689 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 503, "optime" : { "t" : 1358533422000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 503, "optime" : { "t" : 1358533422000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 503, "optime" : { "t" : 1358533462000, "i" : 6400 }, "optimeDate" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 502, "optime" : { "t" : 1358533462000, "i" : 2253 }, "optimeDate" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:23Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 537, "optime" : { "t" : 1358533422000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 502, "optime" : { "t" : 1358533421000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:23Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 502, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:23Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 502, "optime" : { "t" : 1358533462000, "i" : 1831 }, "optimeDate" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 502, "optime" : { "t" : 1358533462000, "i" : 6400 }, "optimeDate" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:23Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 502, "optime" : { "t" : 1358533422000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 536, "optime" : { "t" : 1358533422000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 502, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 502, "optime" : { "t" : 1358533462000, "i" : 6400 }, "optimeDate" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:24Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 503, "optime" : { "t" : 1358533422000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 503, "optime" : { "t" : 1358533421000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:23Z"), "pingMs" : 15 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 535, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 503, "optime" : { "t" : 1358533462000, "i" : 119 }, "optimeDate" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 502, "optime" : { "t" : 1358533462000, "i" : 2096 }, "optimeDate" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:23Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 502, "optime" : { "t" : 1358533421000, "i" : 3309 }, "optimeDate" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 502, "optime" : { "t" : 1358533421000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:23Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 502, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 535, "optime" : { "t" : 1358533464000, "i" : 4689 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:24:25.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 340 more seconds m31003| Fri Jan 18 13:24:25.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 339 more seconds m31003| Fri Jan 18 13:24:25.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 339 more seconds m31003| Fri Jan 18 13:24:25.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 340 more seconds m31003| Fri Jan 18 13:24:25.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 339 more seconds m31003| Fri Jan 18 13:24:25.161 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 339 more seconds m31002| Fri Jan 18 13:24:26.245 [conn264] end connection 127.0.0.5:57259 (5 connections now open) m31000| Fri Jan 18 13:24:26.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31002| Fri Jan 18 13:24:26.399 [initandlisten] connection accepted from 127.0.0.5:58058 #268 (6 connections now open) m31000| Fri Jan 18 13:24:28.037 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:460740 230ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 542, "optime" : { "t" : 1358533469000, "i" : 3261 }, "optimeDate" : ISODate("2013-01-18T18:24:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 508, "optime" : { "t" : 1358533427000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:28Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 508, "optime" : { "t" : 1358533426000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:23:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:29Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 508, "optime" : { "t" : 1358533467000, "i" : 758 }, "optimeDate" : ISODate("2013-01-18T18:24:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:28Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 507, "optime" : { "t" : 1358533468000, "i" : 2396 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 542, "optime" : { "t" : 1358533427000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 507, "optime" : { "t" : 1358533427000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 507, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 507, "optime" : { "t" : 1358533467000, "i" : 5393 }, "optimeDate" : ISODate("2013-01-18T18:24:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 507, "optime" : { "t" : 1358533469000, "i" : 1293 }, "optimeDate" : ISODate("2013-01-18T18:24:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 507, "optime" : { "t" : 1358533427000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:28Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 541, "optime" : { "t" : 1358533427000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 507, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 507, "optime" : { "t" : 1358533469000, "i" : 1034 }, "optimeDate" : ISODate("2013-01-18T18:24:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:29Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 508, "optime" : { "t" : 1358533427000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:28Z"), "pingMs" : 16 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 508, "optime" : { "t" : 1358533427000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:29Z"), "pingMs" : 24 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 540, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 508, "optime" : { "t" : 1358533464000, "i" : 4689 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 507, "optime" : { "t" : 1358533468000, "i" : 2396 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 507, "optime" : { "t" : 1358533427000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:28Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 507, "optime" : { "t" : 1358533427000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-18T18:23:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:29Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 507, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 540, "optime" : { "t" : 1358533469000, "i" : 3261 }, "optimeDate" : ISODate("2013-01-18T18:24:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:24:30.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 335 more seconds m31003| Fri Jan 18 13:24:30.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 334 more seconds m31003| Fri Jan 18 13:24:30.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 334 more seconds m31003| Fri Jan 18 13:24:30.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 335 more seconds m31003| Fri Jan 18 13:24:30.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 334 more seconds m31003| Fri Jan 18 13:24:30.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 334 more seconds m31003| Fri Jan 18 13:24:30.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31004| Fri Jan 18 13:24:33.403 [conn68] end connection 127.0.0.2:41469 (4 connections now open) m31004| Fri Jan 18 13:24:33.403 [initandlisten] connection accepted from 127.0.0.2:39970 #72 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 547, "optime" : { "t" : 1358533474000, "i" : 2477 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 513, "optime" : { "t" : 1358533432000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 513, "optime" : { "t" : 1358533432000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 513, "optime" : { "t" : 1358533473000, "i" : 3875 }, "optimeDate" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 512, "optime" : { "t" : 1358533472000, "i" : 1367 }, "optimeDate" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 547, "optime" : { "t" : 1358533433000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:23:53Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 512, "optime" : { "t" : 1358533431000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:33Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 512, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:33Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 512, "optime" : { "t" : 1358533472000, "i" : 1352 }, "optimeDate" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 512, "optime" : { "t" : 1358533473000, "i" : 1011 }, "optimeDate" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 512, "optime" : { "t" : 1358533432000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 546, "optime" : { "t" : 1358533432000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:52Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 512, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 512, "optime" : { "t" : 1358533473000, "i" : 239 }, "optimeDate" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:34Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 513, "optime" : { "t" : 1358533432000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 16 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 513, "optime" : { "t" : 1358533431000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:33Z"), "pingMs" : 20 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 545, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 513, "optime" : { "t" : 1358533469000, "i" : 3261 }, "optimeDate" : ISODate("2013-01-18T18:24:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 512, "optime" : { "t" : 1358533472000, "i" : 1341 }, "optimeDate" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 512, "optime" : { "t" : 1358533431000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 512, "optime" : { "t" : 1358533431000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:33Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 512, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 545, "optime" : { "t" : 1358533474000, "i" : 2477 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:24:34.360 [conn58] end connection 127.0.0.6:48635 (3 connections now open) m31003| Fri Jan 18 13:24:34.360 [initandlisten] connection accepted from 127.0.0.6:34315 #61 (5 connections now open) m31004| Fri Jan 18 13:24:34.381 [conn69] end connection 127.0.0.3:53124 (4 connections now open) m31004| Fri Jan 18 13:24:34.381 [initandlisten] connection accepted from 127.0.0.3:42612 #73 (5 connections now open) m31004| Fri Jan 18 13:24:35.049 [conn70] end connection 127.0.0.4:55506 (4 connections now open) m31004| Fri Jan 18 13:24:35.049 [initandlisten] connection accepted from 127.0.0.4:55989 #74 (5 connections now open) m31003| Fri Jan 18 13:24:35.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 330 more seconds m31003| Fri Jan 18 13:24:35.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 329 more seconds m31003| Fri Jan 18 13:24:35.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 329 more seconds m31003| Fri Jan 18 13:24:35.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 330 more seconds m31003| Fri Jan 18 13:24:35.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 329 more seconds m31003| Fri Jan 18 13:24:35.162 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 329 more seconds m31002| Fri Jan 18 13:24:35.403 [conn265] end connection 127.0.0.2:45699 (5 connections now open) m31002| Fri Jan 18 13:24:35.404 [initandlisten] connection accepted from 127.0.0.2:39046 #269 (6 connections now open) m31001| Fri Jan 18 13:24:36.360 [conn358] end connection 127.0.0.6:59535 (4 connections now open) m31001| Fri Jan 18 13:24:36.360 [initandlisten] connection accepted from 127.0.0.6:51590 #362 (5 connections now open) m31002| Fri Jan 18 13:24:36.381 [conn266] end connection 127.0.0.3:57754 (5 connections now open) m31002| Fri Jan 18 13:24:36.381 [initandlisten] connection accepted from 127.0.0.3:43069 #270 (6 connections now open) m31001| Fri Jan 18 13:24:37.260 [conn359] end connection 127.0.0.4:58658 (4 connections now open) m31001| Fri Jan 18 13:24:37.260 [initandlisten] connection accepted from 127.0.0.4:57095 #363 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 552, "optime" : { "t" : 1358533479000, "i" : 5968 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 518, "optime" : { "t" : 1358533438000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 518, "optime" : { "t" : 1358533438000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 518, "optime" : { "t" : 1358533479000, "i" : 2715 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 517, "optime" : { "t" : 1358533478000, "i" : 2898 }, "optimeDate" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 552, "optime" : { "t" : 1358533438000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 517, "optime" : { "t" : 1358533437000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:39Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 517, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:39Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 517, "optime" : { "t" : 1358533478000, "i" : 2898 }, "optimeDate" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 517, "optime" : { "t" : 1358533479000, "i" : 702 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 517, "optime" : { "t" : 1358533438000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 551, "optime" : { "t" : 1358533438000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 517, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 517, "optime" : { "t" : 1358533479000, "i" : 677 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:39Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 518, "optime" : { "t" : 1358533438000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 14 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 518, "optime" : { "t" : 1358533437000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:39Z"), "pingMs" : 14 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 550, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 518, "optime" : { "t" : 1358533479000, "i" : 5430 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 517, "optime" : { "t" : 1358533478000, "i" : 2898 }, "optimeDate" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 517, "optime" : { "t" : 1358533437000, "i" : 3556 }, "optimeDate" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:38Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 517, "optime" : { "t" : 1358533437000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:23:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:39Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 517, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:39Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 550, "optime" : { "t" : 1358533479000, "i" : 5430 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:24:40.163 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 325 more seconds m31003| Fri Jan 18 13:24:40.163 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 324 more seconds m31003| Fri Jan 18 13:24:40.163 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 324 more seconds m31003| Fri Jan 18 13:24:40.163 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 325 more seconds m31003| Fri Jan 18 13:24:40.163 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 324 more seconds m31003| Fri Jan 18 13:24:40.163 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 324 more seconds m31003| Fri Jan 18 13:24:40.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:24:41.429 [rsMgr] replSet I don't see a primary and I can't elect myself m31001| Fri Jan 18 13:24:43.429 [conn360] end connection 127.0.0.5:35463 (4 connections now open) m31001| Fri Jan 18 13:24:43.457 [initandlisten] connection accepted from 127.0.0.5:36444 #364 (5 connections now open) m31004| Fri Jan 18 13:24:44.659 [conn71] end connection 127.0.0.5:46083 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 558, "optime" : { "t" : 1358533485000, "i" : 367 }, "optimeDate" : ISODate("2013-01-18T18:24:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 524, "optime" : { "t" : 1358533443000, "i" : 4138 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 524, "optime" : { "t" : 1358533442000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 524, "optime" : { "t" : 1358533483000, "i" : 2860 }, "optimeDate" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:44Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 523, "optime" : { "t" : 1358533484000, "i" : 2678 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 558, "optime" : { "t" : 1358533443000, "i" : 4138 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 523, "optime" : { "t" : 1358533443000, "i" : 4138 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 523, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 523, "optime" : { "t" : 1358533484000, "i" : 2661 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 523, "optime" : { "t" : 1358533483000, "i" : 193 }, "optimeDate" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 523, "optime" : { "t" : 1358533442000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 557, "optime" : { "t" : 1358533443000, "i" : 4138 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 523, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:44Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 523, "optime" : { "t" : 1358533483000, "i" : 187 }, "optimeDate" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:44Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:45Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 524, "optime" : { "t" : 1358533443000, "i" : 4138 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:44Z"), "pingMs" : 33 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 524, "optime" : { "t" : 1358533443000, "i" : 4138 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 35 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 556, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 524, "optime" : { "t" : 1358533479000, "i" : 5968 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:44Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 523, "optime" : { "t" : 1358533484000, "i" : 2678 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 523, "optime" : { "t" : 1358533443000, "i" : 4138 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 523, "optime" : { "t" : 1358533443000, "i" : 4138 }, "optimeDate" : ISODate("2013-01-18T18:24:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 523, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 556, "optime" : { "t" : 1358533485000, "i" : 367 }, "optimeDate" : ISODate("2013-01-18T18:24:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:24:45.160 [initandlisten] connection accepted from 127.0.0.5:37691 #75 (5 connections now open) m31003| Fri Jan 18 13:24:45.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 320 more seconds m31003| Fri Jan 18 13:24:45.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 319 more seconds m31003| Fri Jan 18 13:24:45.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 319 more seconds m31003| Fri Jan 18 13:24:45.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 320 more seconds m31003| Fri Jan 18 13:24:45.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 319 more seconds m31003| Fri Jan 18 13:24:45.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 319 more seconds m31000| Fri Jan 18 13:24:46.441 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:140841 nreturned:160000 reslen:2720020 140ms m31000| Fri Jan 18 13:24:46.772 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:131725 nreturned:160000 reslen:2720020 131ms m31000| Fri Jan 18 13:24:48.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:24:50.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 315 more seconds m31003| Fri Jan 18 13:24:50.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 314 more seconds m31003| Fri Jan 18 13:24:50.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 314 more seconds m31003| Fri Jan 18 13:24:50.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 315 more seconds m31003| Fri Jan 18 13:24:50.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 314 more seconds m31003| Fri Jan 18 13:24:50.164 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 314 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 563, "optime" : { "t" : 1358533490000, "i" : 1990 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 529, "optime" : { "t" : 1358533448000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 529, "optime" : { "t" : 1358533448000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 529, "optime" : { "t" : 1358533489000, "i" : 3257 }, "optimeDate" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 528, "optime" : { "t" : 1358533488000, "i" : 2859 }, "optimeDate" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 563, "optime" : { "t" : 1358533448000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 528, "optime" : { "t" : 1358533447000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:49Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 528, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:49Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 528, "optime" : { "t" : 1358533488000, "i" : 2856 }, "optimeDate" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 528, "optime" : { "t" : 1358533489000, "i" : 217 }, "optimeDate" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 528, "optime" : { "t" : 1358533448000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 562, "optime" : { "t" : 1358533448000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 528, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 528, "optime" : { "t" : 1358533488000, "i" : 8119 }, "optimeDate" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:50Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 529, "optime" : { "t" : 1358533448000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 529, "optime" : { "t" : 1358533447000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:49Z"), "pingMs" : 40 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 561, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 529, "optime" : { "t" : 1358533490000, "i" : 1321 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 528, "optime" : { "t" : 1358533488000, "i" : 2759 }, "optimeDate" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 528, "optime" : { "t" : 1358533447000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 528, "optime" : { "t" : 1358533447000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:49Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 528, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:50Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 561, "optime" : { "t" : 1358533490000, "i" : 1619 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:24:50.362 [conn267] end connection 127.0.0.6:44252 (5 connections now open) m31002| Fri Jan 18 13:24:50.362 [initandlisten] connection accepted from 127.0.0.6:48134 #271 (7 connections now open) m31003| Fri Jan 18 13:24:50.383 [conn59] end connection 127.0.0.3:49280 (3 connections now open) m31003| Fri Jan 18 13:24:50.383 [initandlisten] connection accepted from 127.0.0.3:39325 #62 (4 connections now open) m31003| Fri Jan 18 13:24:51.051 [conn60] end connection 127.0.0.4:37028 (3 connections now open) m31003| Fri Jan 18 13:24:51.052 [initandlisten] connection accepted from 127.0.0.4:48014 #63 (4 connections now open) m31001| Fri Jan 18 13:24:51.406 [conn361] end connection 127.0.0.2:52064 (4 connections now open) m31001| Fri Jan 18 13:24:51.406 [initandlisten] connection accepted from 127.0.0.2:49190 #365 (5 connections now open) m31000| Fri Jan 18 13:24:52.362 [conn81] end connection 127.0.0.6:49398 (11 connections now open) m31000| Fri Jan 18 13:24:52.362 [initandlisten] connection accepted from 127.0.0.6:45536 #84 (12 connections now open) m31000| Fri Jan 18 13:24:52.383 [conn82] end connection 127.0.0.3:56992 (11 connections now open) m31000| Fri Jan 18 13:24:52.383 [initandlisten] connection accepted from 127.0.0.3:55935 #85 (12 connections now open) m31000| Fri Jan 18 13:24:53.052 [conn83] end connection 127.0.0.4:52611 (11 connections now open) m31000| Fri Jan 18 13:24:53.052 [initandlisten] connection accepted from 127.0.0.4:55287 #86 (12 connections now open) m31003| Fri Jan 18 13:24:55.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 310 more seconds m31003| Fri Jan 18 13:24:55.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 309 more seconds m31003| Fri Jan 18 13:24:55.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 309 more seconds m31003| Fri Jan 18 13:24:55.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 310 more seconds m31003| Fri Jan 18 13:24:55.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 309 more seconds m31003| Fri Jan 18 13:24:55.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 309 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 568, "optime" : { "t" : 1358533495000, "i" : 5879 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 534, "optime" : { "t" : 1358533453000, "i" : 6265 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 534, "optime" : { "t" : 1358533453000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 534, "optime" : { "t" : 1358533495000, "i" : 3235 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 533, "optime" : { "t" : 1358533494000, "i" : 2987 }, "optimeDate" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 568, "optime" : { "t" : 1358533453000, "i" : 6265 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 533, "optime" : { "t" : 1358533453000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:55Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 533, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:55Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 533, "optime" : { "t" : 1358533494000, "i" : 2855 }, "optimeDate" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 533, "optime" : { "t" : 1358533495000, "i" : 624 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 533, "optime" : { "t" : 1358533453000, "i" : 6265 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 567, "optime" : { "t" : 1358533453000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 533, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 533, "optime" : { "t" : 1358533495000, "i" : 608 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:55Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 534, "optime" : { "t" : 1358533453000, "i" : 6265 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 33 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 534, "optime" : { "t" : 1358533453000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:55Z"), "pingMs" : 30 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 566, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 534, "optime" : { "t" : 1358533495000, "i" : 1514 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:24:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 533, "optime" : { "t" : 1358533494000, "i" : 2775 }, "optimeDate" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 533, "optime" : { "t" : 1358533453000, "i" : 6265 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 533, "optime" : { "t" : 1358533453000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:55Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 533, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:24:55Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 566, "optime" : { "t" : 1358533495000, "i" : 5879 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:24:56.821 [conn268] end connection 127.0.0.5:58058 (5 connections now open) m31002| Fri Jan 18 13:24:56.845 [initandlisten] connection accepted from 127.0.0.5:39823 #272 (6 connections now open) m31000| Fri Jan 18 13:24:58.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:25:00.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 305 more seconds m31003| Fri Jan 18 13:25:00.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 304 more seconds m31003| Fri Jan 18 13:25:00.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 304 more seconds m31003| Fri Jan 18 13:25:00.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 305 more seconds m31003| Fri Jan 18 13:25:00.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 304 more seconds m31003| Fri Jan 18 13:25:00.165 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 304 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 574, "optime" : { "t" : 1358533501000, "i" : 1169 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 540, "optime" : { "t" : 1358533458000, "i" : 6310 }, "optimeDate" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 540, "optime" : { "t" : 1358533458000, "i" : 6310 }, "optimeDate" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:01Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 540, "optime" : { "t" : 1358533499000, "i" : 3423 }, "optimeDate" : ISODate("2013-01-18T18:24:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 539, "optime" : { "t" : 1358533500000, "i" : 3497 }, "optimeDate" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 574, "optime" : { "t" : 1358533458000, "i" : 6310 }, "optimeDate" : ISODate("2013-01-18T18:24:18Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 539, "optime" : { "t" : 1358533459000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 539, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 539, "optime" : { "t" : 1358533500000, "i" : 3490 }, "optimeDate" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 539, "optime" : { "t" : 1358533501000, "i" : 629 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 539, "optime" : { "t" : 1358533458000, "i" : 6310 }, "optimeDate" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 573, "optime" : { "t" : 1358533459000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:19Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 539, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 539, "optime" : { "t" : 1358533501000, "i" : 575 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:01Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 540, "optime" : { "t" : 1358533458000, "i" : 6310 }, "optimeDate" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:00Z"), "pingMs" : 20 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 540, "optime" : { "t" : 1358533459000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:01Z"), "pingMs" : 18 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 572, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 540, "optime" : { "t" : 1358533500000, "i" : 1599 }, "optimeDate" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 539, "optime" : { "t" : 1358533500000, "i" : 3316 }, "optimeDate" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 539, "optime" : { "t" : 1358533458000, "i" : 6310 }, "optimeDate" : ISODate("2013-01-18T18:24:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 539, "optime" : { "t" : 1358533459000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:01Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 539, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 572, "optime" : { "t" : 1358533501000, "i" : 1169 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:25:02.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31004| Fri Jan 18 13:25:03.408 [conn72] end connection 127.0.0.2:39970 (4 connections now open) m31004| Fri Jan 18 13:25:03.408 [initandlisten] connection accepted from 127.0.0.2:38269 #76 (5 connections now open) m31003| Fri Jan 18 13:25:04.364 [conn61] end connection 127.0.0.6:34315 (3 connections now open) m31003| Fri Jan 18 13:25:04.364 [initandlisten] connection accepted from 127.0.0.6:51203 #64 (4 connections now open) m31004| Fri Jan 18 13:25:04.385 [conn73] end connection 127.0.0.3:42612 (4 connections now open) m31004| Fri Jan 18 13:25:04.385 [initandlisten] connection accepted from 127.0.0.3:34513 #77 (6 connections now open) m31004| Fri Jan 18 13:25:05.053 [conn74] end connection 127.0.0.4:55989 (4 connections now open) m31004| Fri Jan 18 13:25:05.053 [initandlisten] connection accepted from 127.0.0.4:52665 #78 (5 connections now open) m31003| Fri Jan 18 13:25:05.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 300 more seconds m31003| Fri Jan 18 13:25:05.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 299 more seconds m31003| Fri Jan 18 13:25:05.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 299 more seconds m31003| Fri Jan 18 13:25:05.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 300 more seconds m31003| Fri Jan 18 13:25:05.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 299 more seconds m31003| Fri Jan 18 13:25:05.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 299 more seconds m31002| Fri Jan 18 13:25:05.408 [conn269] end connection 127.0.0.2:39046 (5 connections now open) m31002| Fri Jan 18 13:25:05.408 [initandlisten] connection accepted from 127.0.0.2:48521 #273 (6 connections now open) m31003| Fri Jan 18 13:25:05.910 [rsSync] replSet still syncing, not yet to minValid optime 50f991f5:10f5 m31001| Fri Jan 18 13:25:06.364 [conn362] end connection 127.0.0.6:51590 (4 connections now open) m31001| Fri Jan 18 13:25:06.364 [initandlisten] connection accepted from 127.0.0.6:32925 #366 (5 connections now open) m31002| Fri Jan 18 13:25:06.385 [conn270] end connection 127.0.0.3:43069 (5 connections now open) m31002| Fri Jan 18 13:25:06.385 [initandlisten] connection accepted from 127.0.0.3:33015 #274 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 579, "optime" : { "t" : 1358533506000, "i" : 3940 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 545, "optime" : { "t" : 1358533464000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 545, "optime" : { "t" : 1358533464000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 545, "optime" : { "t" : 1358533505000, "i" : 3389 }, "optimeDate" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 544, "optime" : { "t" : 1358533506000, "i" : 3727 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 579, "optime" : { "t" : 1358533464000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 544, "optime" : { "t" : 1358533464000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 544, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 544, "optime" : { "t" : 1358533506000, "i" : 3721 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 544, "optime" : { "t" : 1358533505000, "i" : 622 }, "optimeDate" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 544, "optime" : { "t" : 1358533464000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 578, "optime" : { "t" : 1358533464000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 544, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 544, "optime" : { "t" : 1358533505000, "i" : 28 }, "optimeDate" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:06Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 545, "optime" : { "t" : 1358533464000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:06Z"), "pingMs" : 20 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 545, "optime" : { "t" : 1358533464000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:05Z"), "pingMs" : 17 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 577, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 545, "optime" : { "t" : 1358533505000, "i" : 1184 }, "optimeDate" : ISODate("2013-01-18T18:25:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 544, "optime" : { "t" : 1358533506000, "i" : 3605 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 544, "optime" : { "t" : 1358533464000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 544, "optime" : { "t" : 1358533464000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 544, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 577, "optime" : { "t" : 1358533506000, "i" : 3940 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31001| Fri Jan 18 13:25:07.264 [conn363] end connection 127.0.0.4:57095 (4 connections now open) m31001| Fri Jan 18 13:25:07.265 [initandlisten] connection accepted from 127.0.0.4:49775 #367 (5 connections now open) m31003| Fri Jan 18 13:25:10.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 295 more seconds m31003| Fri Jan 18 13:25:10.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 294 more seconds m31003| Fri Jan 18 13:25:10.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 294 more seconds m31003| Fri Jan 18 13:25:10.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 295 more seconds m31003| Fri Jan 18 13:25:10.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 294 more seconds m31003| Fri Jan 18 13:25:10.166 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 294 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 584, "optime" : { "t" : 1358533511000, "i" : 6185 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 550, "optime" : { "t" : 1358533468000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 550, "optime" : { "t" : 1358533468000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 550, "optime" : { "t" : 1358533511000, "i" : 3361 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 549, "optime" : { "t" : 1358533510000, "i" : 3671 }, "optimeDate" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 584, "optime" : { "t" : 1358533468000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 549, "optime" : { "t" : 1358533468000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:11Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 549, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 549, "optime" : { "t" : 1358533510000, "i" : 3658 }, "optimeDate" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 549, "optime" : { "t" : 1358533511000, "i" : 378 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 549, "optime" : { "t" : 1358533468000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 583, "optime" : { "t" : 1358533468000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 549, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 549, "optime" : { "t" : 1358533511000, "i" : 377 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:11Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 550, "optime" : { "t" : 1358533468000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:10Z"), "pingMs" : 13 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 550, "optime" : { "t" : 1358533468000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:11Z"), "pingMs" : 12 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 582, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 550, "optime" : { "t" : 1358533510000, "i" : 1028 }, "optimeDate" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 549, "optime" : { "t" : 1358533510000, "i" : 3506 }, "optimeDate" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 549, "optime" : { "t" : 1358533468000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 549, "optime" : { "t" : 1358533468000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:24:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:11Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 549, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 582, "optime" : { "t" : 1358533511000, "i" : 5912 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:25:12.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Fri Jan 18 13:25:13.806 [conn364] end connection 127.0.0.5:36444 (4 connections now open) m31001| Fri Jan 18 13:25:13.896 [initandlisten] connection accepted from 127.0.0.5:40127 #368 (5 connections now open) m31003| Fri Jan 18 13:25:15.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 290 more seconds m31003| Fri Jan 18 13:25:15.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 289 more seconds m31003| Fri Jan 18 13:25:15.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 289 more seconds m31003| Fri Jan 18 13:25:15.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 290 more seconds m31003| Fri Jan 18 13:25:15.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 289 more seconds m31003| Fri Jan 18 13:25:15.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 289 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 590, "optime" : { "t" : 1358533517000, "i" : 516 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 556, "optime" : { "t" : 1358533474000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:16Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 556, "optime" : { "t" : 1358533474000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 556, "optime" : { "t" : 1358533515000, "i" : 3314 }, "optimeDate" : ISODate("2013-01-18T18:25:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:16Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 555, "optime" : { "t" : 1358533516000, "i" : 3601 }, "optimeDate" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 590, "optime" : { "t" : 1358533474000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 555, "optime" : { "t" : 1358533474000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 555, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 555, "optime" : { "t" : 1358533516000, "i" : 3311 }, "optimeDate" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 555, "optime" : { "t" : 1358533517000, "i" : 516 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 555, "optime" : { "t" : 1358533474000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:16Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 589, "optime" : { "t" : 1358533474000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 555, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 555, "optime" : { "t" : 1358533517000, "i" : 293 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:17Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 556, "optime" : { "t" : 1358533474000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:16Z"), "pingMs" : 9 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 556, "optime" : { "t" : 1358533474000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:17Z"), "pingMs" : 16 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 588, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 556, "optime" : { "t" : 1358533515000, "i" : 1420 }, "optimeDate" : ISODate("2013-01-18T18:25:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 555, "optime" : { "t" : 1358533516000, "i" : 3451 }, "optimeDate" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 555, "optime" : { "t" : 1358533474000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:16Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 555, "optime" : { "t" : 1358533474000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:17Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 555, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 588, "optime" : { "t" : 1358533517000, "i" : 293 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:25:20.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 285 more seconds m31003| Fri Jan 18 13:25:20.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 284 more seconds m31003| Fri Jan 18 13:25:20.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 284 more seconds m31003| Fri Jan 18 13:25:20.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 285 more seconds m31003| Fri Jan 18 13:25:20.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 284 more seconds m31003| Fri Jan 18 13:25:20.167 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 284 more seconds m31000| Fri Jan 18 13:25:20.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31002| Fri Jan 18 13:25:20.366 [conn271] end connection 127.0.0.6:48134 (5 connections now open) m31002| Fri Jan 18 13:25:20.366 [initandlisten] connection accepted from 127.0.0.6:41550 #275 (6 connections now open) m31003| Fri Jan 18 13:25:20.387 [conn62] end connection 127.0.0.3:39325 (3 connections now open) m31003| Fri Jan 18 13:25:20.387 [initandlisten] connection accepted from 127.0.0.3:54404 #65 (4 connections now open) m31003| Fri Jan 18 13:25:21.056 [conn63] end connection 127.0.0.4:48014 (3 connections now open) m31003| Fri Jan 18 13:25:21.056 [initandlisten] connection accepted from 127.0.0.4:59255 #66 (4 connections now open) m31001| Fri Jan 18 13:25:21.411 [conn365] end connection 127.0.0.2:49190 (4 connections now open) m31001| Fri Jan 18 13:25:21.411 [initandlisten] connection accepted from 127.0.0.2:40236 #369 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:25:22.366 [conn84] end connection 127.0.0.6:45536 (11 connections now open) m31000| Fri Jan 18 13:25:22.366 [initandlisten] connection accepted from 127.0.0.6:38845 #87 (12 connections now open) m31000| Fri Jan 18 13:25:22.387 [conn85] end connection 127.0.0.3:55935 (11 connections now open) m31000| Fri Jan 18 13:25:22.387 [initandlisten] connection accepted from 127.0.0.3:60147 #88 (12 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 595, "optime" : { "t" : 1358533522000, "i" : 2845 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 561, "optime" : { "t" : 1358533479000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 561, "optime" : { "t" : 1358533479000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 561, "optime" : { "t" : 1358533521000, "i" : 3117 }, "optimeDate" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 560, "optime" : { "t" : 1358533522000, "i" : 2745 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 595, "optime" : { "t" : 1358533479000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 560, "optime" : { "t" : 1358533479000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 560, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 560, "optime" : { "t" : 1358533522000, "i" : 2739 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 560, "optime" : { "t" : 1358533521000, "i" : 441 }, "optimeDate" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:21Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 560, "optime" : { "t" : 1358533479000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 594, "optime" : { "t" : 1358533479000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 560, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:21Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 560, "optime" : { "t" : 1358533521000, "i" : 241 }, "optimeDate" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:22Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 561, "optime" : { "t" : 1358533479000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:22Z"), "pingMs" : 4 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 561, "optime" : { "t" : 1358533479000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 23 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 593, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 561, "optime" : { "t" : 1358533520000, "i" : 208 }, "optimeDate" : ISODate("2013-01-18T18:25:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 560, "optime" : { "t" : 1358533522000, "i" : 2739 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 560, "optime" : { "t" : 1358533479000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 560, "optime" : { "t" : 1358533479000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 560, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 593, "optime" : { "t" : 1358533522000, "i" : 2845 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:25:22.664 [conn75] end connection 127.0.0.5:37691 (4 connections now open) m31000| Fri Jan 18 13:25:23.056 [conn86] end connection 127.0.0.4:55287 (11 connections now open) m31000| Fri Jan 18 13:25:23.056 [initandlisten] connection accepted from 127.0.0.4:53654 #89 (12 connections now open) m31004| Fri Jan 18 13:25:23.164 [initandlisten] connection accepted from 127.0.0.5:34927 #79 (5 connections now open) m31003| Fri Jan 18 13:25:25.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 280 more seconds m31003| Fri Jan 18 13:25:25.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 279 more seconds m31003| Fri Jan 18 13:25:25.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 279 more seconds m31003| Fri Jan 18 13:25:25.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 280 more seconds m31003| Fri Jan 18 13:25:25.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 279 more seconds m31003| Fri Jan 18 13:25:25.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 279 more seconds m31002| Fri Jan 18 13:25:27.100 [conn272] end connection 127.0.0.5:39823 (5 connections now open) m31002| Fri Jan 18 13:25:27.238 [initandlisten] connection accepted from 127.0.0.5:49486 #276 (6 connections now open) m31000| Fri Jan 18 13:25:27.518 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:381603 190ms STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:25:27.766 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:287719 143ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 600, "optime" : { "t" : 1358533527000, "i" : 6528 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 566, "optime" : { "t" : 1358533484000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 566, "optime" : { "t" : 1358533484000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 566, "optime" : { "t" : 1358533527000, "i" : 1526 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 565, "optime" : { "t" : 1358533526000, "i" : 3323 }, "optimeDate" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 600, "optime" : { "t" : 1358533484000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 565, "optime" : { "t" : 1358533484000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:27Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 565, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 565, "optime" : { "t" : 1358533526000, "i" : 2692 }, "optimeDate" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 565, "optime" : { "t" : 1358533527000, "i" : 240 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 565, "optime" : { "t" : 1358533484000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 599, "optime" : { "t" : 1358533484000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 565, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:27Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 565, "optime" : { "t" : 1358533527000, "i" : 240 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:27Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 566, "optime" : { "t" : 1358533484000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:26Z"), "pingMs" : 2 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 566, "optime" : { "t" : 1358533484000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 27 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 598, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 566, "optime" : { "t" : 1358533525000, "i" : 1710 }, "optimeDate" : ISODate("2013-01-18T18:25:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 565, "optime" : { "t" : 1358533526000, "i" : 2880 }, "optimeDate" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 565, "optime" : { "t" : 1358533484000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:26Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 565, "optime" : { "t" : 1358533484000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:27Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 565, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 598, "optime" : { "t" : 1358533527000, "i" : 6528 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:25:28.335 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 numYields: 449 locks(micros) r:223719 nreturned:160000 reslen:2720020 165ms m31000| Fri Jan 18 13:25:28.431 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:161177 nreturned:160000 reslen:2720020 161ms m31003| Fri Jan 18 13:25:30.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 275 more seconds m31003| Fri Jan 18 13:25:30.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 274 more seconds m31003| Fri Jan 18 13:25:30.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 274 more seconds m31003| Fri Jan 18 13:25:30.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 275 more seconds m31003| Fri Jan 18 13:25:30.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 274 more seconds m31003| Fri Jan 18 13:25:30.168 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 274 more seconds m31000| Fri Jan 18 13:25:30.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:25:33.113 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:344787 172ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 606, "optime" : { "t" : 1358533533000, "i" : 1891 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 572, "optime" : { "t" : 1358533490000, "i" : 112 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 572, "optime" : { "t" : 1358533490000, "i" : 112 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:33Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 572, "optime" : { "t" : 1358533531000, "i" : 3579 }, "optimeDate" : ISODate("2013-01-18T18:25:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:32Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 571, "optime" : { "t" : 1358533532000, "i" : 3406 }, "optimeDate" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 606, "optime" : { "t" : 1358533490000, "i" : 112 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 571, "optime" : { "t" : 1358533490000, "i" : 112 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 571, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 571, "optime" : { "t" : 1358533532000, "i" : 3406 }, "optimeDate" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 571, "optime" : { "t" : 1358533532000, "i" : 8168 }, "optimeDate" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 571, "optime" : { "t" : 1358533490000, "i" : 112 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 605, "optime" : { "t" : 1358533490000, "i" : 112 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 571, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 571, "optime" : { "t" : 1358533532000, "i" : 7911 }, "optimeDate" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:33Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 572, "optime" : { "t" : 1358533490000, "i" : 112 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:32Z"), "pingMs" : 14 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 572, "optime" : { "t" : 1358533490000, "i" : 112 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:33Z"), "pingMs" : 32 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 604, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 572, "optime" : { "t" : 1358533533000, "i" : 1320 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 571, "optime" : { "t" : 1358533532000, "i" : 3406 }, "optimeDate" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 571, "optime" : { "t" : 1358533490000, "i" : 112 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 571, "optime" : { "t" : 1358533490000, "i" : 112 }, "optimeDate" : ISODate("2013-01-18T18:24:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:33Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 571, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:33Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 604, "optime" : { "t" : 1358533533000, "i" : 1891 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:25:33.412 [conn76] end connection 127.0.0.2:38269 (4 connections now open) m31004| Fri Jan 18 13:25:33.413 [initandlisten] connection accepted from 127.0.0.2:41439 #80 (5 connections now open) m31003| Fri Jan 18 13:25:34.368 [conn64] end connection 127.0.0.6:51203 (3 connections now open) m31003| Fri Jan 18 13:25:34.368 [initandlisten] connection accepted from 127.0.0.6:45100 #67 (4 connections now open) m31004| Fri Jan 18 13:25:34.388 [conn77] end connection 127.0.0.3:34513 (4 connections now open) m31004| Fri Jan 18 13:25:34.388 [initandlisten] connection accepted from 127.0.0.3:55174 #81 (5 connections now open) m31003| Fri Jan 18 13:25:34.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31004| Fri Jan 18 13:25:35.057 [conn78] end connection 127.0.0.4:52665 (4 connections now open) m31004| Fri Jan 18 13:25:35.058 [initandlisten] connection accepted from 127.0.0.4:47089 #82 (5 connections now open) m31003| Fri Jan 18 13:25:35.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 270 more seconds m31003| Fri Jan 18 13:25:35.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 269 more seconds m31003| Fri Jan 18 13:25:35.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 269 more seconds m31003| Fri Jan 18 13:25:35.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 270 more seconds m31003| Fri Jan 18 13:25:35.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 269 more seconds m31003| Fri Jan 18 13:25:35.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 269 more seconds m31002| Fri Jan 18 13:25:35.413 [conn273] end connection 127.0.0.2:48521 (5 connections now open) m31002| Fri Jan 18 13:25:35.413 [initandlisten] connection accepted from 127.0.0.2:37210 #277 (6 connections now open) m31001| Fri Jan 18 13:25:36.368 [conn366] end connection 127.0.0.6:32925 (4 connections now open) m31001| Fri Jan 18 13:25:36.368 [initandlisten] connection accepted from 127.0.0.6:41289 #370 (6 connections now open) m31002| Fri Jan 18 13:25:36.388 [conn274] end connection 127.0.0.3:33015 (5 connections now open) m31002| Fri Jan 18 13:25:36.389 [initandlisten] connection accepted from 127.0.0.3:41341 #278 (6 connections now open) m31001| Fri Jan 18 13:25:37.269 [conn367] end connection 127.0.0.4:49775 (4 connections now open) m31001| Fri Jan 18 13:25:37.269 [initandlisten] connection accepted from 127.0.0.4:39985 #371 (5 connections now open) m31000| Fri Jan 18 13:25:37.442 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:227114 113ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 611, "optime" : { "t" : 1358533538000, "i" : 5554 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 577, "optime" : { "t" : 1358533495000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:38Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 577, "optime" : { "t" : 1358533495000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 577, "optime" : { "t" : 1358533537000, "i" : 2700 }, "optimeDate" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:38Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 576, "optime" : { "t" : 1358533538000, "i" : 5124 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 611, "optime" : { "t" : 1358533498000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:58Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 576, "optime" : { "t" : 1358533498000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 576, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 576, "optime" : { "t" : 1358533538000, "i" : 5069 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 576, "optime" : { "t" : 1358533537000, "i" : 78 }, "optimeDate" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:37Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 576, "optime" : { "t" : 1358533495000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:38Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 610, "optime" : { "t" : 1358533498000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:58Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 576, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:37Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 576, "optime" : { "t" : 1358533537000, "i" : 78 }, "optimeDate" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:38Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:38Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 577, "optime" : { "t" : 1358533495000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:38Z"), "pingMs" : 13 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 577, "optime" : { "t" : 1358533495000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:24:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 39 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 609, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 577, "optime" : { "t" : 1358533538000, "i" : 2007 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 576, "optime" : { "t" : 1358533538000, "i" : 4865 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 576, "optime" : { "t" : 1358533498000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:38Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 576, "optime" : { "t" : 1358533498000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:24:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 576, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 609, "optime" : { "t" : 1358533538000, "i" : 5554 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:25:40.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 265 more seconds m31003| Fri Jan 18 13:25:40.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 264 more seconds m31003| Fri Jan 18 13:25:40.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 264 more seconds m31003| Fri Jan 18 13:25:40.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 265 more seconds m31003| Fri Jan 18 13:25:40.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 264 more seconds m31003| Fri Jan 18 13:25:40.169 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 264 more seconds m31000| Fri Jan 18 13:25:40.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 616, "optime" : { "t" : 1358533543000, "i" : 6240 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 582, "optime" : { "t" : 1358533501000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 582, "optime" : { "t" : 1358533501000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 582, "optime" : { "t" : 1358533543000, "i" : 3339 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 581, "optime" : { "t" : 1358533542000, "i" : 3618 }, "optimeDate" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 616, "optime" : { "t" : 1358533501000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 581, "optime" : { "t" : 1358533501000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:43Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 581, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 581, "optime" : { "t" : 1358533542000, "i" : 3602 }, "optimeDate" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 581, "optime" : { "t" : 1358533543000, "i" : 199 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 581, "optime" : { "t" : 1358533501000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 615, "optime" : { "t" : 1358533501000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 581, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:43Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 581, "optime" : { "t" : 1358533543000, "i" : 183 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:43Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 582, "optime" : { "t" : 1358533501000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:42Z"), "pingMs" : 15 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 582, "optime" : { "t" : 1358533501000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 24 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 614, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 582, "optime" : { "t" : 1358533543000, "i" : 1299 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 581, "optime" : { "t" : 1358533542000, "i" : 3467 }, "optimeDate" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 581, "optime" : { "t" : 1358533501000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:42Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 581, "optime" : { "t" : 1358533501000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:43Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 581, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:43Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 614, "optime" : { "t" : 1358533543000, "i" : 6240 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31001| Fri Jan 18 13:25:44.113 [conn368] end connection 127.0.0.5:40127 (4 connections now open) m31001| Fri Jan 18 13:25:44.200 [initandlisten] connection accepted from 127.0.0.5:51555 #372 (5 connections now open) m31003| Fri Jan 18 13:25:44.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:25:45.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 260 more seconds m31003| Fri Jan 18 13:25:45.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 259 more seconds m31003| Fri Jan 18 13:25:45.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 259 more seconds m31003| Fri Jan 18 13:25:45.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 260 more seconds m31003| Fri Jan 18 13:25:45.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 259 more seconds m31003| Fri Jan 18 13:25:45.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 259 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 622, "optime" : { "t" : 1358533549000, "i" : 506 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 588, "optime" : { "t" : 1358533506000, "i" : 1307 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 588, "optime" : { "t" : 1358533506000, "i" : 1307 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:49Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 588, "optime" : { "t" : 1358533547000, "i" : 3741 }, "optimeDate" : ISODate("2013-01-18T18:25:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:48Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 587, "optime" : { "t" : 1358533548000, "i" : 3692 }, "optimeDate" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 622, "optime" : { "t" : 1358533506000, "i" : 1307 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 587, "optime" : { "t" : 1358533506000, "i" : 1307 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 587, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 587, "optime" : { "t" : 1358533548000, "i" : 3522 }, "optimeDate" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 587, "optime" : { "t" : 1358533549000, "i" : 248 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 587, "optime" : { "t" : 1358533506000, "i" : 1307 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 621, "optime" : { "t" : 1358533509000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:09Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 587, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 587, "optime" : { "t" : 1358533549000, "i" : 247 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:49Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 588, "optime" : { "t" : 1358533506000, "i" : 1307 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:48Z"), "pingMs" : 42 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 588, "optime" : { "t" : 1358533506000, "i" : 1307 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:49Z"), "pingMs" : 15 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 620, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 588, "optime" : { "t" : 1358533548000, "i" : 1541 }, "optimeDate" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 587, "optime" : { "t" : 1358533548000, "i" : 3541 }, "optimeDate" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 587, "optime" : { "t" : 1358533506000, "i" : 1307 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 587, "optime" : { "t" : 1358533506000, "i" : 1307 }, "optimeDate" : ISODate("2013-01-18T18:25:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:49Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 587, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 620, "optime" : { "t" : 1358533549000, "i" : 506 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:25:50.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 255 more seconds m31003| Fri Jan 18 13:25:50.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 254 more seconds m31003| Fri Jan 18 13:25:50.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 254 more seconds m31003| Fri Jan 18 13:25:50.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 255 more seconds m31003| Fri Jan 18 13:25:50.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 254 more seconds m31003| Fri Jan 18 13:25:50.170 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 254 more seconds m31002| Fri Jan 18 13:25:50.370 [conn275] end connection 127.0.0.6:41550 (5 connections now open) m31002| Fri Jan 18 13:25:50.370 [initandlisten] connection accepted from 127.0.0.6:36784 #279 (6 connections now open) m31003| Fri Jan 18 13:25:50.391 [conn65] end connection 127.0.0.3:54404 (3 connections now open) m31003| Fri Jan 18 13:25:50.391 [initandlisten] connection accepted from 127.0.0.3:59874 #68 (4 connections now open) m31003| Fri Jan 18 13:25:51.060 [conn66] end connection 127.0.0.4:59255 (3 connections now open) m31003| Fri Jan 18 13:25:51.060 [initandlisten] connection accepted from 127.0.0.4:38150 #69 (4 connections now open) m31001| Fri Jan 18 13:25:51.415 [conn369] end connection 127.0.0.2:40236 (4 connections now open) m31001| Fri Jan 18 13:25:51.415 [initandlisten] connection accepted from 127.0.0.2:45012 #373 (6 connections now open) m31000| Fri Jan 18 13:25:52.370 [conn87] end connection 127.0.0.6:38845 (11 connections now open) m31000| Fri Jan 18 13:25:52.370 [initandlisten] connection accepted from 127.0.0.6:47315 #90 (12 connections now open) m31000| Fri Jan 18 13:25:52.391 [conn88] end connection 127.0.0.3:60147 (11 connections now open) m31000| Fri Jan 18 13:25:52.391 [initandlisten] connection accepted from 127.0.0.3:56974 #91 (13 connections now open) m31000| Fri Jan 18 13:25:53.061 [conn89] end connection 127.0.0.4:53654 (11 connections now open) m31000| Fri Jan 18 13:25:53.061 [initandlisten] connection accepted from 127.0.0.4:51402 #92 (12 connections now open) m31003| Fri Jan 18 13:25:54.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 627, "optime" : { "t" : 1358533554000, "i" : 5180 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 593, "optime" : { "t" : 1358533511000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 593, "optime" : { "t" : 1358533511000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 593, "optime" : { "t" : 1358533553000, "i" : 3561 }, "optimeDate" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:54Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 592, "optime" : { "t" : 1358533554000, "i" : 3347 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 627, "optime" : { "t" : 1358533511000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 592, "optime" : { "t" : 1358533511000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 592, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 592, "optime" : { "t" : 1358533554000, "i" : 2535 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 592, "optime" : { "t" : 1358533553000, "i" : 523 }, "optimeDate" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:53Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 592, "optime" : { "t" : 1358533511000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 626, "optime" : { "t" : 1358533511000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 592, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:53Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 592, "optime" : { "t" : 1358533552000, "i" : 6234 }, "optimeDate" : ISODate("2013-01-18T18:25:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:54Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:54Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 593, "optime" : { "t" : 1358533511000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:54Z"), "pingMs" : 20 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 593, "optime" : { "t" : 1358533511000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 26 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 625, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 593, "optime" : { "t" : 1358533553000, "i" : 1183 }, "optimeDate" : ISODate("2013-01-18T18:25:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:25:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 592, "optime" : { "t" : 1358533554000, "i" : 3148 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 592, "optime" : { "t" : 1358533511000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 592, "optime" : { "t" : 1358533511000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 592, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 625, "optime" : { "t" : 1358533554000, "i" : 5112 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:25:55.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 250 more seconds m31003| Fri Jan 18 13:25:55.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 249 more seconds m31003| Fri Jan 18 13:25:55.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 249 more seconds m31003| Fri Jan 18 13:25:55.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 250 more seconds m31003| Fri Jan 18 13:25:55.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 249 more seconds m31003| Fri Jan 18 13:25:55.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 249 more seconds m31002| Fri Jan 18 13:25:57.768 [conn276] end connection 127.0.0.5:49486 (5 connections now open) m31002| Fri Jan 18 13:25:57.841 [initandlisten] connection accepted from 127.0.0.5:41914 #280 (6 connections now open) m31003| Fri Jan 18 13:25:58.169 [rsMgr] replSet I don't see a primary and I can't elect myself STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 633, "optime" : { "t" : 1358533560000, "i" : 677 }, "optimeDate" : ISODate("2013-01-18T18:26:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 599, "optime" : { "t" : 1358533517000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 599, "optime" : { "t" : 1358533517000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 599, "optime" : { "t" : 1358533559000, "i" : 3697 }, "optimeDate" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 598, "optime" : { "t" : 1358533558000, "i" : 3184 }, "optimeDate" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 633, "optime" : { "t" : 1358533517000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 598, "optime" : { "t" : 1358533517000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:59Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 598, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:58Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 598, "optime" : { "t" : 1358533558000, "i" : 3134 }, "optimeDate" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 598, "optime" : { "t" : 1358533559000, "i" : 355 }, "optimeDate" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 598, "optime" : { "t" : 1358533517000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 632, "optime" : { "t" : 1358533517000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 598, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:59Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 598, "optime" : { "t" : 1358533559000, "i" : 351 }, "optimeDate" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:00Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 599, "optime" : { "t" : 1358533517000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 40 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 599, "optime" : { "t" : 1358533517000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 29 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 631, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 599, "optime" : { "t" : 1358533558000, "i" : 1206 }, "optimeDate" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 598, "optime" : { "t" : 1358533558000, "i" : 2767 }, "optimeDate" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 598, "optime" : { "t" : 1358533517000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:58Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 598, "optime" : { "t" : 1358533517000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:25:59Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 598, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 631, "optime" : { "t" : 1358533560000, "i" : 677 }, "optimeDate" : ISODate("2013-01-18T18:26:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:26:00.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 245 more seconds m31003| Fri Jan 18 13:26:00.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 244 more seconds m31003| Fri Jan 18 13:26:00.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 244 more seconds m31003| Fri Jan 18 13:26:00.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 245 more seconds m31003| Fri Jan 18 13:26:00.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 244 more seconds m31003| Fri Jan 18 13:26:00.171 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 244 more seconds m31004| Fri Jan 18 13:26:00.669 [conn79] end connection 127.0.0.5:34927 (4 connections now open) m31004| Fri Jan 18 13:26:01.169 [initandlisten] connection accepted from 127.0.0.5:36285 #83 (5 connections now open) m31000| Fri Jan 18 13:26:02.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31004| Fri Jan 18 13:26:03.417 [conn80] end connection 127.0.0.2:41439 (4 connections now open) m31004| Fri Jan 18 13:26:03.417 [initandlisten] connection accepted from 127.0.0.2:53381 #84 (5 connections now open) m31003| Fri Jan 18 13:26:04.372 [conn67] end connection 127.0.0.6:45100 (3 connections now open) m31003| Fri Jan 18 13:26:04.372 [initandlisten] connection accepted from 127.0.0.6:39616 #70 (4 connections now open) m31004| Fri Jan 18 13:26:04.392 [conn81] end connection 127.0.0.3:55174 (4 connections now open) m31004| Fri Jan 18 13:26:04.392 [initandlisten] connection accepted from 127.0.0.3:37927 #85 (5 connections now open) m31004| Fri Jan 18 13:26:05.062 [conn82] end connection 127.0.0.4:47089 (4 connections now open) m31004| Fri Jan 18 13:26:05.062 [initandlisten] connection accepted from 127.0.0.4:57452 #86 (5 connections now open) m31003| Fri Jan 18 13:26:05.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 240 more seconds m31003| Fri Jan 18 13:26:05.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 239 more seconds m31003| Fri Jan 18 13:26:05.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 239 more seconds m31003| Fri Jan 18 13:26:05.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 240 more seconds m31003| Fri Jan 18 13:26:05.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 239 more seconds m31003| Fri Jan 18 13:26:05.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 239 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 638, "optime" : { "t" : 1358533565000, "i" : 3001 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 604, "optime" : { "t" : 1358533522000, "i" : 677 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:04Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 604, "optime" : { "t" : 1358533522000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:05Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 604, "optime" : { "t" : 1358533563000, "i" : 3395 }, "optimeDate" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:04Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 603, "optime" : { "t" : 1358533564000, "i" : 3918 }, "optimeDate" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 638, "optime" : { "t" : 1358533525000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:25Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 603, "optime" : { "t" : 1358533522000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:05Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 603, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:04Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 603, "optime" : { "t" : 1358533564000, "i" : 3906 }, "optimeDate" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 603, "optime" : { "t" : 1358533565000, "i" : 921 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 603, "optime" : { "t" : 1358533522000, "i" : 677 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 637, "optime" : { "t" : 1358533522000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 603, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 603, "optime" : { "t" : 1358533565000, "i" : 916 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:05Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 604, "optime" : { "t" : 1358533522000, "i" : 677 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 25 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 604, "optime" : { "t" : 1358533522000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:05Z"), "pingMs" : 46 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 636, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 604, "optime" : { "t" : 1358533563000, "i" : 5413 }, "optimeDate" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 603, "optime" : { "t" : 1358533564000, "i" : 3747 }, "optimeDate" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 603, "optime" : { "t" : 1358533522000, "i" : 677 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:04Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 603, "optime" : { "t" : 1358533522000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:05Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 603, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 636, "optime" : { "t" : 1358533565000, "i" : 3001 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:26:05.417 [conn277] end connection 127.0.0.2:37210 (5 connections now open) m31002| Fri Jan 18 13:26:05.417 [initandlisten] connection accepted from 127.0.0.2:40018 #281 (6 connections now open) m31003| Fri Jan 18 13:26:05.918 [rsSync] replSet still syncing, not yet to minValid optime 50f991f5:10f5 m31001| Fri Jan 18 13:26:06.372 [conn370] end connection 127.0.0.6:41289 (4 connections now open) m31001| Fri Jan 18 13:26:06.372 [initandlisten] connection accepted from 127.0.0.6:50317 #374 (5 connections now open) m31002| Fri Jan 18 13:26:06.393 [conn278] end connection 127.0.0.3:41341 (5 connections now open) m31002| Fri Jan 18 13:26:06.393 [initandlisten] connection accepted from 127.0.0.3:45111 #282 (6 connections now open) m31001| Fri Jan 18 13:26:07.274 [conn371] end connection 127.0.0.4:39985 (4 connections now open) m31001| Fri Jan 18 13:26:07.274 [initandlisten] connection accepted from 127.0.0.4:54110 #375 (5 connections now open) m31000| Fri Jan 18 13:26:08.872 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:830634 415ms m31003| Fri Jan 18 13:26:10.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 235 more seconds m31003| Fri Jan 18 13:26:10.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 234 more seconds m31003| Fri Jan 18 13:26:10.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 234 more seconds m31003| Fri Jan 18 13:26:10.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 235 more seconds m31003| Fri Jan 18 13:26:10.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 234 more seconds m31003| Fri Jan 18 13:26:10.172 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 234 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 643, "optime" : { "t" : 1358533570000, "i" : 4214 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 609, "optime" : { "t" : 1358533527000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 609, "optime" : { "t" : 1358533527000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 609, "optime" : { "t" : 1358533569000, "i" : 6441 }, "optimeDate" : ISODate("2013-01-18T18:26:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:10Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 608, "optime" : { "t" : 1358533570000, "i" : 3527 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 643, "optime" : { "t" : 1358533527000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 608, "optime" : { "t" : 1358533527000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 608, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 608, "optime" : { "t" : 1358533570000, "i" : 3508 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 608, "optime" : { "t" : 1358533569000, "i" : 370 }, "optimeDate" : ISODate("2013-01-18T18:26:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 608, "optime" : { "t" : 1358533527000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 642, "optime" : { "t" : 1358533527000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 608, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:10Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 608, "optime" : { "t" : 1358533569000, "i" : 347 }, "optimeDate" : ISODate("2013-01-18T18:26:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:10Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:10Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 609, "optime" : { "t" : 1358533527000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:10Z"), "pingMs" : 25 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 609, "optime" : { "t" : 1358533527000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 36 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 641, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 609, "optime" : { "t" : 1358533568000, "i" : 725 }, "optimeDate" : ISODate("2013-01-18T18:26:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 608, "optime" : { "t" : 1358533570000, "i" : 3379 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 608, "optime" : { "t" : 1358533527000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 608, "optime" : { "t" : 1358533527000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 608, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 641, "optime" : { "t" : 1358533570000, "i" : 4214 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:26:11.057 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:133042 nreturned:160000 reslen:2720020 133ms m31000| Fri Jan 18 13:26:11.057 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:134889 nreturned:160000 reslen:2720020 134ms m31000| Fri Jan 18 13:26:12.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31001| Fri Jan 18 13:26:14.590 [conn372] end connection 127.0.0.5:51555 (4 connections now open) m31001| Fri Jan 18 13:26:14.663 [initandlisten] connection accepted from 127.0.0.5:53260 #376 (5 connections now open) m31003| Fri Jan 18 13:26:15.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 230 more seconds m31003| Fri Jan 18 13:26:15.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 229 more seconds m31003| Fri Jan 18 13:26:15.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 229 more seconds m31003| Fri Jan 18 13:26:15.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 230 more seconds m31003| Fri Jan 18 13:26:15.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 229 more seconds m31003| Fri Jan 18 13:26:15.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 229 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 648, "optime" : { "t" : 1358533575000, "i" : 5781 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 614, "optime" : { "t" : 1358533533000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 614, "optime" : { "t" : 1358533533000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 614, "optime" : { "t" : 1358533575000, "i" : 3504 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 613, "optime" : { "t" : 1358533574000, "i" : 3408 }, "optimeDate" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 648, "optime" : { "t" : 1358533533000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 613, "optime" : { "t" : 1358533533000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:15Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 613, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:14Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 613, "optime" : { "t" : 1358533574000, "i" : 3408 }, "optimeDate" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 613, "optime" : { "t" : 1358533575000, "i" : 83 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 613, "optime" : { "t" : 1358533533000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 647, "optime" : { "t" : 1358533533000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 613, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 613, "optime" : { "t" : 1358533575000, "i" : 83 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:15Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 614, "optime" : { "t" : 1358533533000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 25 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 614, "optime" : { "t" : 1358533533000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:15Z"), "pingMs" : 53 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 646, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 614, "optime" : { "t" : 1358533573000, "i" : 1194 }, "optimeDate" : ISODate("2013-01-18T18:26:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 613, "optime" : { "t" : 1358533574000, "i" : 3408 }, "optimeDate" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 613, "optime" : { "t" : 1358533533000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:14Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 613, "optime" : { "t" : 1358533533000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:15Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 613, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 646, "optime" : { "t" : 1358533575000, "i" : 5471 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:26:16.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:26:20.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 225 more seconds m31003| Fri Jan 18 13:26:20.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 224 more seconds m31003| Fri Jan 18 13:26:20.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 224 more seconds m31003| Fri Jan 18 13:26:20.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 225 more seconds m31003| Fri Jan 18 13:26:20.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 224 more seconds m31003| Fri Jan 18 13:26:20.173 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 224 more seconds m31002| Fri Jan 18 13:26:20.374 [conn279] end connection 127.0.0.6:36784 (5 connections now open) m31002| Fri Jan 18 13:26:20.374 [initandlisten] connection accepted from 127.0.0.6:37045 #283 (6 connections now open) m31003| Fri Jan 18 13:26:20.395 [conn68] end connection 127.0.0.3:59874 (3 connections now open) m31003| Fri Jan 18 13:26:20.395 [initandlisten] connection accepted from 127.0.0.3:52353 #71 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 654, "optime" : { "t" : 1358533581000, "i" : 73 }, "optimeDate" : ISODate("2013-01-18T18:26:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 620, "optime" : { "t" : 1358533538000, "i" : 2922 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:20Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 620, "optime" : { "t" : 1358533538000, "i" : 2922 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 620, "optime" : { "t" : 1358533579000, "i" : 3983 }, "optimeDate" : ISODate("2013-01-18T18:26:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:20Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 619, "optime" : { "t" : 1358533580000, "i" : 3353 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 654, "optime" : { "t" : 1358533538000, "i" : 2922 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 619, "optime" : { "t" : 1358533538000, "i" : 2922 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 619, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:20Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 619, "optime" : { "t" : 1358533580000, "i" : 3231 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 619, "optime" : { "t" : 1358533579000, "i" : 1019 }, "optimeDate" : ISODate("2013-01-18T18:26:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:19Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 619, "optime" : { "t" : 1358533538000, "i" : 2922 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:20Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 653, "optime" : { "t" : 1358533538000, "i" : 2922 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 619, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:20Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 619, "optime" : { "t" : 1358533579000, "i" : 1011 }, "optimeDate" : ISODate("2013-01-18T18:26:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:20Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:21Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 620, "optime" : { "t" : 1358533538000, "i" : 2922 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 29 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 620, "optime" : { "t" : 1358533538000, "i" : 2922 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 31 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 652, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 620, "optime" : { "t" : 1358533575000, "i" : 5781 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 619, "optime" : { "t" : 1358533580000, "i" : 3271 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 619, "optime" : { "t" : 1358533538000, "i" : 2922 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:20Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 619, "optime" : { "t" : 1358533538000, "i" : 2922 }, "optimeDate" : ISODate("2013-01-18T18:25:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 619, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 652, "optime" : { "t" : 1358533581000, "i" : 73 }, "optimeDate" : ISODate("2013-01-18T18:26:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:26:21.065 [conn69] end connection 127.0.0.4:38150 (3 connections now open) m31003| Fri Jan 18 13:26:21.065 [initandlisten] connection accepted from 127.0.0.4:60290 #72 (4 connections now open) m31001| Fri Jan 18 13:26:21.419 [conn373] end connection 127.0.0.2:45012 (4 connections now open) m31001| Fri Jan 18 13:26:21.420 [initandlisten] connection accepted from 127.0.0.2:60897 #377 (5 connections now open) m31000| Fri Jan 18 13:26:22.374 [conn90] end connection 127.0.0.6:47315 (11 connections now open) m31000| Fri Jan 18 13:26:22.374 [initandlisten] connection accepted from 127.0.0.6:60916 #93 (12 connections now open) m31000| Fri Jan 18 13:26:22.395 [conn91] end connection 127.0.0.3:56974 (11 connections now open) m31000| Fri Jan 18 13:26:22.395 [initandlisten] connection accepted from 127.0.0.3:54594 #94 (12 connections now open) m31000| Fri Jan 18 13:26:23.065 [conn92] end connection 127.0.0.4:51402 (11 connections now open) m31000| Fri Jan 18 13:26:23.065 [initandlisten] connection accepted from 127.0.0.4:36050 #95 (12 connections now open) m31003| Fri Jan 18 13:26:25.174 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 220 more seconds m31003| Fri Jan 18 13:26:25.174 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 219 more seconds m31003| Fri Jan 18 13:26:25.174 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 219 more seconds m31003| Fri Jan 18 13:26:25.174 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 220 more seconds m31003| Fri Jan 18 13:26:25.174 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 219 more seconds m31003| Fri Jan 18 13:26:25.174 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 219 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 659, "optime" : { "t" : 1358533586000, "i" : 2857 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 625, "optime" : { "t" : 1358533543000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 625, "optime" : { "t" : 1358533543000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 625, "optime" : { "t" : 1358533585000, "i" : 3715 }, "optimeDate" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 624, "optime" : { "t" : 1358533584000, "i" : 3018 }, "optimeDate" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:25Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 659, "optime" : { "t" : 1358533543000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 624, "optime" : { "t" : 1358533543000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:25Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 624, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:24Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 624, "optime" : { "t" : 1358533584000, "i" : 3018 }, "optimeDate" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 624, "optime" : { "t" : 1358533585000, "i" : 780 }, "optimeDate" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:25Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 624, "optime" : { "t" : 1358533543000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 658, "optime" : { "t" : 1358533543000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 624, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 624, "optime" : { "t" : 1358533585000, "i" : 15 }, "optimeDate" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:26Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 625, "optime" : { "t" : 1358533543000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 25 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 625, "optime" : { "t" : 1358533543000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:25Z"), "pingMs" : 31 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 657, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 625, "optime" : { "t" : 1358533586000, "i" : 1101 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 624, "optime" : { "t" : 1358533584000, "i" : 2916 }, "optimeDate" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:25Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 624, "optime" : { "t" : 1358533543000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:24Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 624, "optime" : { "t" : 1358533543000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:25Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 624, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:26Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 657, "optime" : { "t" : 1358533586000, "i" : 2714 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:26:26.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Fri Jan 18 13:26:28.474 [conn280] end connection 127.0.0.5:41914 (5 connections now open) m31002| Fri Jan 18 13:26:28.507 [initandlisten] connection accepted from 127.0.0.5:54537 #284 (6 connections now open) m31000| Fri Jan 18 13:26:30.032 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:271324 135ms m31003| Fri Jan 18 13:26:30.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 215 more seconds m31003| Fri Jan 18 13:26:30.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 214 more seconds m31003| Fri Jan 18 13:26:30.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 214 more seconds m31003| Fri Jan 18 13:26:30.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 215 more seconds m31003| Fri Jan 18 13:26:30.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 214 more seconds m31003| Fri Jan 18 13:26:30.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 214 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 664, "optime" : { "t" : 1358533591000, "i" : 4562 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 630, "optime" : { "t" : 1358533549000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 630, "optime" : { "t" : 1358533549000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 630, "optime" : { "t" : 1358533591000, "i" : 3671 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 629, "optime" : { "t" : 1358533590000, "i" : 4919 }, "optimeDate" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 664, "optime" : { "t" : 1358533549000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 629, "optime" : { "t" : 1358533549000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:31Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 629, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:30Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 629, "optime" : { "t" : 1358533590000, "i" : 4879 }, "optimeDate" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 629, "optime" : { "t" : 1358533591000, "i" : 933 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 629, "optime" : { "t" : 1358533549000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 663, "optime" : { "t" : 1358533549000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 629, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 629, "optime" : { "t" : 1358533591000, "i" : 914 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:31Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 630, "optime" : { "t" : 1358533549000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 23 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 630, "optime" : { "t" : 1358533549000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:31Z"), "pingMs" : 23 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 662, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 630, "optime" : { "t" : 1358533591000, "i" : 1567 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 629, "optime" : { "t" : 1358533590000, "i" : 4811 }, "optimeDate" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 629, "optime" : { "t" : 1358533549000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:30Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 629, "optime" : { "t" : 1358533549000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:25:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:31Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 629, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:31Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 662, "optime" : { "t" : 1358533591000, "i" : 4562 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:26:33.421 [conn84] end connection 127.0.0.2:53381 (4 connections now open) m31004| Fri Jan 18 13:26:33.421 [initandlisten] connection accepted from 127.0.0.2:36807 #87 (5 connections now open) m31000| Fri Jan 18 13:26:34.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:26:34.376 [conn70] end connection 127.0.0.6:39616 (3 connections now open) m31003| Fri Jan 18 13:26:34.376 [initandlisten] connection accepted from 127.0.0.6:43927 #73 (4 connections now open) m31004| Fri Jan 18 13:26:34.396 [conn85] end connection 127.0.0.3:37927 (4 connections now open) m31004| Fri Jan 18 13:26:34.396 [initandlisten] connection accepted from 127.0.0.3:46592 #88 (5 connections now open) m31004| Fri Jan 18 13:26:35.067 [conn86] end connection 127.0.0.4:57452 (4 connections now open) m31004| Fri Jan 18 13:26:35.067 [initandlisten] connection accepted from 127.0.0.4:33858 #89 (5 connections now open) m31003| Fri Jan 18 13:26:35.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 210 more seconds m31003| Fri Jan 18 13:26:35.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 209 more seconds m31003| Fri Jan 18 13:26:35.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 209 more seconds m31003| Fri Jan 18 13:26:35.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 210 more seconds m31003| Fri Jan 18 13:26:35.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 209 more seconds m31003| Fri Jan 18 13:26:35.175 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 209 more seconds m31002| Fri Jan 18 13:26:35.422 [conn281] end connection 127.0.0.2:40018 (5 connections now open) m31002| Fri Jan 18 13:26:35.422 [initandlisten] connection accepted from 127.0.0.2:60133 #285 (6 connections now open) m31001| Fri Jan 18 13:26:36.376 [conn374] end connection 127.0.0.6:50317 (4 connections now open) m31001| Fri Jan 18 13:26:36.376 [initandlisten] connection accepted from 127.0.0.6:45176 #378 (5 connections now open) m31002| Fri Jan 18 13:26:36.397 [conn282] end connection 127.0.0.3:45111 (5 connections now open) m31002| Fri Jan 18 13:26:36.397 [initandlisten] connection accepted from 127.0.0.3:50466 #286 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 669, "optime" : { "t" : 1358533596000, "i" : 7331 }, "optimeDate" : ISODate("2013-01-18T18:26:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 635, "optime" : { "t" : 1358533554000, "i" : 1417 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:36Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 635, "optime" : { "t" : 1358533554000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 635, "optime" : { "t" : 1358533595000, "i" : 3199 }, "optimeDate" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:36Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 634, "optime" : { "t" : 1358533596000, "i" : 3846 }, "optimeDate" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 669, "optime" : { "t" : 1358533554000, "i" : 1417 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 634, "optime" : { "t" : 1358533554000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 634, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 634, "optime" : { "t" : 1358533596000, "i" : 3842 }, "optimeDate" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 634, "optime" : { "t" : 1358533595000, "i" : 460 }, "optimeDate" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:35Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 634, "optime" : { "t" : 1358533554000, "i" : 1417 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:36Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 668, "optime" : { "t" : 1358533554000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 634, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:36Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 634, "optime" : { "t" : 1358533595000, "i" : 449 }, "optimeDate" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:36Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:36Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 635, "optime" : { "t" : 1358533554000, "i" : 1417 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:36Z"), "pingMs" : 20 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 635, "optime" : { "t" : 1358533554000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 11 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 667, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 635, "optime" : { "t" : 1358533596000, "i" : 1255 }, "optimeDate" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 634, "optime" : { "t" : 1358533596000, "i" : 3635 }, "optimeDate" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 634, "optime" : { "t" : 1358533554000, "i" : 1417 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:36Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 634, "optime" : { "t" : 1358533554000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 634, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 667, "optime" : { "t" : 1358533596000, "i" : 7331 }, "optimeDate" : ISODate("2013-01-18T18:26:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31001| Fri Jan 18 13:26:37.279 [conn375] end connection 127.0.0.4:54110 (4 connections now open) m31001| Fri Jan 18 13:26:37.279 [initandlisten] connection accepted from 127.0.0.4:46563 #379 (5 connections now open) m31004| Fri Jan 18 13:26:38.674 [conn83] end connection 127.0.0.5:36285 (4 connections now open) m31004| Fri Jan 18 13:26:39.174 [initandlisten] connection accepted from 127.0.0.5:40152 #90 (5 connections now open) m31003| Fri Jan 18 13:26:40.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 205 more seconds m31003| Fri Jan 18 13:26:40.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 204 more seconds m31003| Fri Jan 18 13:26:40.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 204 more seconds m31003| Fri Jan 18 13:26:40.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 205 more seconds m31003| Fri Jan 18 13:26:40.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 204 more seconds m31003| Fri Jan 18 13:26:40.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 204 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 675, "optime" : { "t" : 1358533602000, "i" : 1531 }, "optimeDate" : ISODate("2013-01-18T18:26:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 641, "optime" : { "t" : 1358533560000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 641, "optime" : { "t" : 1358533560000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 641, "optime" : { "t" : 1358533601000, "i" : 3055 }, "optimeDate" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 640, "optime" : { "t" : 1358533600000, "i" : 3439 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:41Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 675, "optime" : { "t" : 1358533560000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:00Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 640, "optime" : { "t" : 1358533559000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:41Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 640, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:40Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 640, "optime" : { "t" : 1358533600000, "i" : 3204 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 640, "optime" : { "t" : 1358533601000, "i" : 497 }, "optimeDate" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:41Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 640, "optime" : { "t" : 1358533560000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 674, "optime" : { "t" : 1358533560000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:00Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 640, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 640, "optime" : { "t" : 1358533601000, "i" : 493 }, "optimeDate" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:42Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 641, "optime" : { "t" : 1358533560000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 12 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 641, "optime" : { "t" : 1358533559000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:41Z"), "pingMs" : 26 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 673, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 641, "optime" : { "t" : 1358533601000, "i" : 5723 }, "optimeDate" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 640, "optime" : { "t" : 1358533600000, "i" : 3289 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:41Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 640, "optime" : { "t" : 1358533559000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 640, "optime" : { "t" : 1358533559000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:25:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:41Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 640, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:41Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 673, "optime" : { "t" : 1358533602000, "i" : 1531 }, "optimeDate" : ISODate("2013-01-18T18:26:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:26:44.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31001| Fri Jan 18 13:26:44.965 [conn376] end connection 127.0.0.5:53260 (4 connections now open) m31001| Fri Jan 18 13:26:44.982 [initandlisten] connection accepted from 127.0.0.5:38087 #380 (5 connections now open) m31003| Fri Jan 18 13:26:45.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 200 more seconds m31003| Fri Jan 18 13:26:45.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 199 more seconds m31003| Fri Jan 18 13:26:45.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 199 more seconds m31003| Fri Jan 18 13:26:45.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 200 more seconds m31003| Fri Jan 18 13:26:45.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 199 more seconds m31003| Fri Jan 18 13:26:45.176 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 199 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 680, "optime" : { "t" : 1358533607000, "i" : 3977 }, "optimeDate" : ISODate("2013-01-18T18:26:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 646, "optime" : { "t" : 1358533565000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 646, "optime" : { "t" : 1358533565000, "i" : 1274 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 646, "optime" : { "t" : 1358533607000, "i" : 3827 }, "optimeDate" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 645, "optime" : { "t" : 1358533606000, "i" : 3924 }, "optimeDate" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 680, "optime" : { "t" : 1358533565000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 645, "optime" : { "t" : 1358533565000, "i" : 1274 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:47Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 645, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:46Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 645, "optime" : { "t" : 1358533606000, "i" : 3909 }, "optimeDate" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 645, "optime" : { "t" : 1358533607000, "i" : 909 }, "optimeDate" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 645, "optime" : { "t" : 1358533565000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 679, "optime" : { "t" : 1358533565000, "i" : 1274 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 645, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 645, "optime" : { "t" : 1358533607000, "i" : 901 }, "optimeDate" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:47Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 646, "optime" : { "t" : 1358533565000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 12 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 646, "optime" : { "t" : 1358533565000, "i" : 1274 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:47Z"), "pingMs" : 12 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 678, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 646, "optime" : { "t" : 1358533606000, "i" : 5630 }, "optimeDate" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 645, "optime" : { "t" : 1358533606000, "i" : 3772 }, "optimeDate" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 645, "optime" : { "t" : 1358533565000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:46Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 645, "optime" : { "t" : 1358533565000, "i" : 1274 }, "optimeDate" : ISODate("2013-01-18T18:26:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:47Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 645, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:46Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 678, "optime" : { "t" : 1358533607000, "i" : 3827 }, "optimeDate" : ISODate("2013-01-18T18:26:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:26:48.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:26:50.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 195 more seconds m31003| Fri Jan 18 13:26:50.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 194 more seconds m31003| Fri Jan 18 13:26:50.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 194 more seconds m31003| Fri Jan 18 13:26:50.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 195 more seconds m31003| Fri Jan 18 13:26:50.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 194 more seconds m31003| Fri Jan 18 13:26:50.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 194 more seconds m31002| Fri Jan 18 13:26:50.378 [conn283] end connection 127.0.0.6:37045 (5 connections now open) m31002| Fri Jan 18 13:26:50.378 [initandlisten] connection accepted from 127.0.0.6:49189 #287 (7 connections now open) m31003| Fri Jan 18 13:26:50.399 [conn71] end connection 127.0.0.3:52353 (3 connections now open) m31003| Fri Jan 18 13:26:50.399 [initandlisten] connection accepted from 127.0.0.3:58764 #74 (4 connections now open) m31003| Fri Jan 18 13:26:51.070 [conn72] end connection 127.0.0.4:60290 (3 connections now open) m31003| Fri Jan 18 13:26:51.070 [initandlisten] connection accepted from 127.0.0.4:44319 #75 (4 connections now open) m31001| Fri Jan 18 13:26:51.424 [conn377] end connection 127.0.0.2:60897 (4 connections now open) m31001| Fri Jan 18 13:26:51.424 [initandlisten] connection accepted from 127.0.0.2:53725 #381 (5 connections now open) m31000| Fri Jan 18 13:26:52.378 [conn93] end connection 127.0.0.6:60916 (11 connections now open) m31000| Fri Jan 18 13:26:52.378 [initandlisten] connection accepted from 127.0.0.6:36569 #96 (12 connections now open) m31000| Fri Jan 18 13:26:52.399 [conn94] end connection 127.0.0.3:54594 (11 connections now open) m31000| Fri Jan 18 13:26:52.399 [initandlisten] connection accepted from 127.0.0.3:49263 #97 (12 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 685, "optime" : { "t" : 1358533612000, "i" : 6422 }, "optimeDate" : ISODate("2013-01-18T18:26:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 651, "optime" : { "t" : 1358533570000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:52Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 651, "optime" : { "t" : 1358533570000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 651, "optime" : { "t" : 1358533611000, "i" : 3467 }, "optimeDate" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 650, "optime" : { "t" : 1358533612000, "i" : 3284 }, "optimeDate" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 685, "optime" : { "t" : 1358533570000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 650, "optime" : { "t" : 1358533570000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 650, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 650, "optime" : { "t" : 1358533612000, "i" : 3284 }, "optimeDate" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 650, "optime" : { "t" : 1358533610000, "i" : 4447 }, "optimeDate" : ISODate("2013-01-18T18:26:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:51Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 650, "optime" : { "t" : 1358533570000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:52Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 684, "optime" : { "t" : 1358533570000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 650, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:52Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 650, "optime" : { "t" : 1358533610000, "i" : 4447 }, "optimeDate" : ISODate("2013-01-18T18:26:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:52Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 651, "optime" : { "t" : 1358533570000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:52Z"), "pingMs" : 7 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 651, "optime" : { "t" : 1358533570000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 7 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 683, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 651, "optime" : { "t" : 1358533611000, "i" : 6127 }, "optimeDate" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 650, "optime" : { "t" : 1358533612000, "i" : 3284 }, "optimeDate" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 650, "optime" : { "t" : 1358533570000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:52Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 650, "optime" : { "t" : 1358533570000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 650, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 683, "optime" : { "t" : 1358533612000, "i" : 6422 }, "optimeDate" : ISODate("2013-01-18T18:26:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:26:53.070 [conn95] end connection 127.0.0.4:36050 (11 connections now open) m31000| Fri Jan 18 13:26:53.070 [initandlisten] connection accepted from 127.0.0.4:52655 #98 (12 connections now open) m31000| Fri Jan 18 13:26:54.287 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:225287 nreturned:160000 reslen:2720020 154ms m31000| Fri Jan 18 13:26:54.312 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 numYields: 2 locks(micros) r:289841 nreturned:160000 reslen:2720020 147ms m31003| Fri Jan 18 13:26:55.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 190 more seconds m31003| Fri Jan 18 13:26:55.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 189 more seconds m31003| Fri Jan 18 13:26:55.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 189 more seconds m31003| Fri Jan 18 13:26:55.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 190 more seconds m31003| Fri Jan 18 13:26:55.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 189 more seconds m31003| Fri Jan 18 13:26:55.177 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 189 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 690, "optime" : { "t" : 1358533617000, "i" : 8270 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 656, "optime" : { "t" : 1358533575000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 656, "optime" : { "t" : 1358533575000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 656, "optime" : { "t" : 1358533617000, "i" : 3539 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 655, "optime" : { "t" : 1358533616000, "i" : 3790 }, "optimeDate" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 690, "optime" : { "t" : 1358533575000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 655, "optime" : { "t" : 1358533575000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:57Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 655, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:56Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 655, "optime" : { "t" : 1358533616000, "i" : 3694 }, "optimeDate" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 655, "optime" : { "t" : 1358533617000, "i" : 208 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 655, "optime" : { "t" : 1358533575000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 689, "optime" : { "t" : 1358533575000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 655, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 655, "optime" : { "t" : 1358533617000, "i" : 202 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:57Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 656, "optime" : { "t" : 1358533575000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 4 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 656, "optime" : { "t" : 1358533575000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:57Z"), "pingMs" : 4 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 688, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 656, "optime" : { "t" : 1358533616000, "i" : 6481 }, "optimeDate" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:26:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 655, "optime" : { "t" : 1358533616000, "i" : 3653 }, "optimeDate" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 655, "optime" : { "t" : 1358533575000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:56Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 655, "optime" : { "t" : 1358533575000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:57Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 655, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:26:56Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 688, "optime" : { "t" : 1358533617000, "i" : 8270 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:26:58.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Fri Jan 18 13:26:58.642 [conn284] end connection 127.0.0.5:54537 (5 connections now open) m31002| Fri Jan 18 13:26:58.696 [initandlisten] connection accepted from 127.0.0.5:41408 #288 (6 connections now open) m31003| Fri Jan 18 13:27:00.178 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 185 more seconds m31003| Fri Jan 18 13:27:00.178 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 184 more seconds m31003| Fri Jan 18 13:27:00.178 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 184 more seconds m31003| Fri Jan 18 13:27:00.178 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 185 more seconds m31003| Fri Jan 18 13:27:00.178 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 184 more seconds m31003| Fri Jan 18 13:27:00.178 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 184 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 696, "optime" : { "t" : 1358533623000, "i" : 1091 }, "optimeDate" : ISODate("2013-01-18T18:27:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 662, "optime" : { "t" : 1358533580000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:02Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 662, "optime" : { "t" : 1358533580000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:03Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 662, "optime" : { "t" : 1358533621000, "i" : 3623 }, "optimeDate" : ISODate("2013-01-18T18:27:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:02Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 661, "optime" : { "t" : 1358533622000, "i" : 3362 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 696, "optime" : { "t" : 1358533580000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 661, "optime" : { "t" : 1358533580000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 661, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:02Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 661, "optime" : { "t" : 1358533622000, "i" : 2766 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 661, "optime" : { "t" : 1358533623000, "i" : 545 }, "optimeDate" : ISODate("2013-01-18T18:27:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 661, "optime" : { "t" : 1358533580000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:02Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 695, "optime" : { "t" : 1358533580000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 661, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 661, "optime" : { "t" : 1358533623000, "i" : 488 }, "optimeDate" : ISODate("2013-01-18T18:27:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:03Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 662, "optime" : { "t" : 1358533580000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 1 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 662, "optime" : { "t" : 1358533580000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:03Z"), "pingMs" : 14 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 694, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 662, "optime" : { "t" : 1358533621000, "i" : 6754 }, "optimeDate" : ISODate("2013-01-18T18:27:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 661, "optime" : { "t" : 1358533622000, "i" : 3179 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 661, "optime" : { "t" : 1358533580000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:02Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 661, "optime" : { "t" : 1358533580000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:03Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 661, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 694, "optime" : { "t" : 1358533623000, "i" : 1091 }, "optimeDate" : ISODate("2013-01-18T18:27:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:27:03.426 [conn87] end connection 127.0.0.2:36807 (4 connections now open) m31004| Fri Jan 18 13:27:03.426 [initandlisten] connection accepted from 127.0.0.2:54458 #91 (5 connections now open) m31003| Fri Jan 18 13:27:04.380 [conn73] end connection 127.0.0.6:43927 (3 connections now open) m31003| Fri Jan 18 13:27:04.380 [initandlisten] connection accepted from 127.0.0.6:54008 #76 (4 connections now open) m31004| Fri Jan 18 13:27:04.400 [conn88] end connection 127.0.0.3:46592 (4 connections now open) m31004| Fri Jan 18 13:27:04.401 [initandlisten] connection accepted from 127.0.0.3:56359 #92 (5 connections now open) m31004| Fri Jan 18 13:27:05.072 [conn89] end connection 127.0.0.4:33858 (4 connections now open) m31004| Fri Jan 18 13:27:05.072 [initandlisten] connection accepted from 127.0.0.4:45172 #93 (5 connections now open) m31003| Fri Jan 18 13:27:05.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 180 more seconds m31003| Fri Jan 18 13:27:05.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 179 more seconds m31003| Fri Jan 18 13:27:05.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 179 more seconds m31003| Fri Jan 18 13:27:05.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 180 more seconds m31003| Fri Jan 18 13:27:05.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 179 more seconds m31003| Fri Jan 18 13:27:05.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 179 more seconds m31002| Fri Jan 18 13:27:05.426 [conn285] end connection 127.0.0.2:60133 (5 connections now open) m31002| Fri Jan 18 13:27:05.426 [initandlisten] connection accepted from 127.0.0.2:48284 #289 (6 connections now open) m31003| Fri Jan 18 13:27:05.925 [rsSync] replSet still syncing, not yet to minValid optime 50f991f5:10f5 m31000| Fri Jan 18 13:27:06.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31001| Fri Jan 18 13:27:06.380 [conn378] end connection 127.0.0.6:45176 (4 connections now open) m31001| Fri Jan 18 13:27:06.380 [initandlisten] connection accepted from 127.0.0.6:36370 #382 (5 connections now open) m31002| Fri Jan 18 13:27:06.401 [conn286] end connection 127.0.0.3:50466 (5 connections now open) m31002| Fri Jan 18 13:27:06.401 [initandlisten] connection accepted from 127.0.0.3:42734 #290 (6 connections now open) m31001| Fri Jan 18 13:27:07.284 [conn379] end connection 127.0.0.4:46563 (4 connections now open) m31001| Fri Jan 18 13:27:07.284 [initandlisten] connection accepted from 127.0.0.4:49518 #383 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 701, "optime" : { "t" : 1358533628000, "i" : 4246 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 667, "optime" : { "t" : 1358533586000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 667, "optime" : { "t" : 1358533586000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 667, "optime" : { "t" : 1358533627000, "i" : 3936 }, "optimeDate" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 666, "optime" : { "t" : 1358533628000, "i" : 3124 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 701, "optime" : { "t" : 1358533586000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 666, "optime" : { "t" : 1358533586000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 666, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 666, "optime" : { "t" : 1358533628000, "i" : 3115 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 666, "optime" : { "t" : 1358533627000, "i" : 767 }, "optimeDate" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:07Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 666, "optime" : { "t" : 1358533586000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 700, "optime" : { "t" : 1358533586000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 666, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 666, "optime" : { "t" : 1358533627000, "i" : 698 }, "optimeDate" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:08Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 667, "optime" : { "t" : 1358533586000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 667, "optime" : { "t" : 1358533585000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:07Z"), "pingMs" : 16 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 699, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 667, "optime" : { "t" : 1358533626000, "i" : 4469 }, "optimeDate" : ISODate("2013-01-18T18:27:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 666, "optime" : { "t" : 1358533628000, "i" : 3124 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 666, "optime" : { "t" : 1358533586000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:08Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 666, "optime" : { "t" : 1358533586000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 666, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 699, "optime" : { "t" : 1358533628000, "i" : 4246 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:27:10.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 175 more seconds m31003| Fri Jan 18 13:27:10.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 174 more seconds m31003| Fri Jan 18 13:27:10.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 174 more seconds m31003| Fri Jan 18 13:27:10.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 175 more seconds m31003| Fri Jan 18 13:27:10.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 174 more seconds m31003| Fri Jan 18 13:27:10.179 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 174 more seconds m31000| Fri Jan 18 13:27:13.598 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:788678 394ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 706, "optime" : { "t" : 1358533633000, "i" : 6621 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 672, "optime" : { "t" : 1358533591000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 672, "optime" : { "t" : 1358533591000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 672, "optime" : { "t" : 1358533633000, "i" : 1589 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 671, "optime" : { "t" : 1358533632000, "i" : 3558 }, "optimeDate" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 706, "optime" : { "t" : 1358533591000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 671, "optime" : { "t" : 1358533591000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:13Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 671, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:13Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 671, "optime" : { "t" : 1358533632000, "i" : 3495 }, "optimeDate" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 671, "optime" : { "t" : 1358533633000, "i" : 426 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 671, "optime" : { "t" : 1358533591000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 705, "optime" : { "t" : 1358533591000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 671, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 671, "optime" : { "t" : 1358533633000, "i" : 424 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:13Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 672, "optime" : { "t" : 1358533591000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 57 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 672, "optime" : { "t" : 1358533591000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:13Z"), "pingMs" : 25 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 704, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 672, "optime" : { "t" : 1358533631000, "i" : 1507 }, "optimeDate" : ISODate("2013-01-18T18:27:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 671, "optime" : { "t" : 1358533632000, "i" : 2942 }, "optimeDate" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 671, "optime" : { "t" : 1358533591000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:12Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 671, "optime" : { "t" : 1358533591000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:13Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 671, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 704, "optime" : { "t" : 1358533633000, "i" : 6295 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:27:14.178 [rsMgr] replSet I don't see a primary and I can't elect myself m31001| Fri Jan 18 13:27:15.106 [conn380] end connection 127.0.0.5:38087 (4 connections now open) m31001| Fri Jan 18 13:27:15.178 [initandlisten] connection accepted from 127.0.0.5:56301 #384 (5 connections now open) m31003| Fri Jan 18 13:27:15.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 170 more seconds m31003| Fri Jan 18 13:27:15.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 169 more seconds m31003| Fri Jan 18 13:27:15.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 169 more seconds m31003| Fri Jan 18 13:27:15.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 170 more seconds m31003| Fri Jan 18 13:27:15.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 169 more seconds m31003| Fri Jan 18 13:27:15.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 169 more seconds m31000| Fri Jan 18 13:27:16.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31004| Fri Jan 18 13:27:16.679 [conn90] end connection 127.0.0.5:40152 (4 connections now open) m31004| Fri Jan 18 13:27:17.179 [initandlisten] connection accepted from 127.0.0.5:48621 #94 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:27:19.296 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.4, filling with zeroes... m31003| Fri Jan 18 13:27:20.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 165 more seconds m31003| Fri Jan 18 13:27:20.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 164 more seconds m31003| Fri Jan 18 13:27:20.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 164 more seconds m31003| Fri Jan 18 13:27:20.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 165 more seconds m31003| Fri Jan 18 13:27:20.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 164 more seconds m31003| Fri Jan 18 13:27:20.180 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 164 more seconds m31002| Fri Jan 18 13:27:20.382 [conn287] end connection 127.0.0.6:49189 (5 connections now open) m31002| Fri Jan 18 13:27:20.382 [initandlisten] connection accepted from 127.0.0.6:36928 #291 (6 connections now open) m31003| Fri Jan 18 13:27:20.403 [conn74] end connection 127.0.0.3:58764 (3 connections now open) m31003| Fri Jan 18 13:27:20.403 [initandlisten] connection accepted from 127.0.0.3:44591 #77 (4 connections now open) m31003| Fri Jan 18 13:27:20.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:27:21.074 [conn75] end connection 127.0.0.4:44319 (3 connections now open) m31003| Fri Jan 18 13:27:21.074 [initandlisten] connection accepted from 127.0.0.4:34467 #78 (4 connections now open) m31000| Fri Jan 18 13:27:21.331 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.4, size: 256MB, took 2.034 secs m31000| Fri Jan 18 13:27:21.331 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:2035214 2035ms m31004| Fri Jan 18 13:27:21.332 [FileAllocator] allocating new datafile /data/db/testReplSet-4/foo.4, filling with zeroes... m31001| Fri Jan 18 13:27:21.429 [conn381] end connection 127.0.0.2:53725 (4 connections now open) m31001| Fri Jan 18 13:27:21.429 [initandlisten] connection accepted from 127.0.0.2:59505 #385 (5 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 714, "optime" : { "t" : 1358533641000, "i" : 6393 }, "optimeDate" : ISODate("2013-01-18T18:27:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 680, "optime" : { "t" : 1358533600000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 680, "optime" : { "t" : 1358533600000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:26:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 680, "optime" : { "t" : 1358533639000, "i" : 2253 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 679, "optime" : { "t" : 1358533639000, "i" : 2253 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:21Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 714, "optime" : { "t" : 1358533601000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:41Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 679, "optime" : { "t" : 1358533600000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:21Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 679, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:21Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 679, "optime" : { "t" : 1358533639000, "i" : 2253 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 679, "optime" : { "t" : 1358533639000, "i" : 2253 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:21Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 679, "optime" : { "t" : 1358533600000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 713, "optime" : { "t" : 1358533600000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 679, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 679, "optime" : { "t" : 1358533639000, "i" : 2253 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:21Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 680, "optime" : { "t" : 1358533600000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 30 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 680, "optime" : { "t" : 1358533600000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:21Z"), "pingMs" : 26 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 712, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 680, "optime" : { "t" : 1358533639000, "i" : 2253 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 679, "optime" : { "t" : 1358533639000, "i" : 2253 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:21Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 679, "optime" : { "t" : 1358533600000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:20Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 679, "optime" : { "t" : 1358533599000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:21Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 679, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 712, "optime" : { "t" : 1358533639000, "i" : 2253 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:27:22.382 [conn96] end connection 127.0.0.6:36569 (11 connections now open) m31000| Fri Jan 18 13:27:22.382 [initandlisten] connection accepted from 127.0.0.6:42909 #99 (12 connections now open) m31000| Fri Jan 18 13:27:22.403 [conn97] end connection 127.0.0.3:49263 (11 connections now open) m31000| Fri Jan 18 13:27:22.403 [initandlisten] connection accepted from 127.0.0.3:44130 #100 (12 connections now open) m31000| Fri Jan 18 13:27:23.074 [conn98] end connection 127.0.0.4:52655 (11 connections now open) m31000| Fri Jan 18 13:27:23.075 [initandlisten] connection accepted from 127.0.0.4:33457 #101 (12 connections now open) m31004| Fri Jan 18 13:27:23.475 [FileAllocator] done allocating datafile /data/db/testReplSet-4/foo.4, size: 256MB, took 2.142 secs m31003| Fri Jan 18 13:27:25.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 160 more seconds m31003| Fri Jan 18 13:27:25.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 159 more seconds m31003| Fri Jan 18 13:27:25.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 159 more seconds m31003| Fri Jan 18 13:27:25.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 160 more seconds m31003| Fri Jan 18 13:27:25.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 159 more seconds m31003| Fri Jan 18 13:27:25.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 159 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 720, "optime" : { "t" : 1358533647000, "i" : 420 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 686, "optime" : { "t" : 1358533602000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:26Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 686, "optime" : { "t" : 1358533602000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:27Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 686, "optime" : { "t" : 1358533645000, "i" : 3690 }, "optimeDate" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:26Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 685, "optime" : { "t" : 1358533646000, "i" : 3029 }, "optimeDate" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 720, "optime" : { "t" : 1358533606000, "i" : 7770 }, "optimeDate" : ISODate("2013-01-18T18:26:46Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 685, "optime" : { "t" : 1358533605000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 685, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 685, "optime" : { "t" : 1358533646000, "i" : 3019 }, "optimeDate" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 685, "optime" : { "t" : 1358533647000, "i" : 212 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 685, "optime" : { "t" : 1358533602000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:26Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 719, "optime" : { "t" : 1358533606000, "i" : 7770 }, "optimeDate" : ISODate("2013-01-18T18:26:46Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 685, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 685, "optime" : { "t" : 1358533647000, "i" : 196 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:27Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 686, "optime" : { "t" : 1358533602000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:26Z"), "pingMs" : 29 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 686, "optime" : { "t" : 1358533602000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:27Z"), "pingMs" : 21 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 718, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 686, "optime" : { "t" : 1358533641000, "i" : 6393 }, "optimeDate" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 685, "optime" : { "t" : 1358533646000, "i" : 3029 }, "optimeDate" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 685, "optime" : { "t" : 1358533606000, "i" : 2769 }, "optimeDate" : ISODate("2013-01-18T18:26:46Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:26Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 685, "optime" : { "t" : 1358533605000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:26:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:27Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 685, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 718, "optime" : { "t" : 1358533647000, "i" : 420 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:27:29.101 [conn288] end connection 127.0.0.5:41408 (5 connections now open) m31002| Fri Jan 18 13:27:29.101 [initandlisten] connection accepted from 127.0.0.5:42430 #292 (6 connections now open) m31003| Fri Jan 18 13:27:30.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 155 more seconds m31003| Fri Jan 18 13:27:30.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 154 more seconds m31003| Fri Jan 18 13:27:30.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 154 more seconds m31003| Fri Jan 18 13:27:30.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 155 more seconds m31003| Fri Jan 18 13:27:30.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 154 more seconds m31003| Fri Jan 18 13:27:30.181 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 154 more seconds m31003| Fri Jan 18 13:27:30.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Fri Jan 18 13:27:32.646 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:4913651 2456ms m31000| Fri Jan 18 13:27:32.646 [conn24] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185723707855858 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:45 nreturned:2 reslen:226 2457ms m31000| Fri Jan 18 13:27:32.646 [conn26] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185723782124859 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:36 nreturned:2 reslen:226 2457ms m31000| Fri Jan 18 13:27:32.646 [conn23] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185725031169493 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:55 nreturned:6 reslen:638 2458ms m31000| Fri Jan 18 13:27:33.097 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:694984 347ms STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:27:33.321 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:238994 119ms m31004| Fri Jan 18 13:27:33.431 [conn91] end connection 127.0.0.2:54458 (4 connections now open) m31004| Fri Jan 18 13:27:33.431 [initandlisten] connection accepted from 127.0.0.2:48541 #95 (5 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 727, "optime" : { "t" : 1358533654000, "i" : 2467 }, "optimeDate" : ISODate("2013-01-18T18:27:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 693, "optime" : { "t" : 1358533611000, "i" : 554 }, "optimeDate" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 693, "optime" : { "t" : 1358533611000, "i" : 554 }, "optimeDate" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 693, "optime" : { "t" : 1358533653000, "i" : 4829 }, "optimeDate" : ISODate("2013-01-18T18:27:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 692, "optime" : { "t" : 1358533650000, "i" : 727 }, "optimeDate" : ISODate("2013-01-18T18:27:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 727, "optime" : { "t" : 1358533612000, "i" : 1426 }, "optimeDate" : ISODate("2013-01-18T18:26:52Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 692, "optime" : { "t" : 1358533607000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:33Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 692, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:33Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 692, "optime" : { "t" : 1358533650000, "i" : 711 }, "optimeDate" : ISODate("2013-01-18T18:27:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 692, "optime" : { "t" : 1358533652000, "i" : 2603 }, "optimeDate" : ISODate("2013-01-18T18:27:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 692, "optime" : { "t" : 1358533610000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 726, "optime" : { "t" : 1358533612000, "i" : 1426 }, "optimeDate" : ISODate("2013-01-18T18:26:52Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 692, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:33Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 692, "optime" : { "t" : 1358533652000, "i" : 2577 }, "optimeDate" : ISODate("2013-01-18T18:27:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:34Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 693, "optime" : { "t" : 1358533611000, "i" : 5555 }, "optimeDate" : ISODate("2013-01-18T18:26:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 40 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 693, "optime" : { "t" : 1358533610000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 19 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 725, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 693, "optime" : { "t" : 1358533650000, "i" : 711 }, "optimeDate" : ISODate("2013-01-18T18:27:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 692, "optime" : { "t" : 1358533650000, "i" : 727 }, "optimeDate" : ISODate("2013-01-18T18:27:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 692, "optime" : { "t" : 1358533607000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 692, "optime" : { "t" : 1358533607000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:26:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:33Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 692, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 725, "optime" : { "t" : 1358533654000, "i" : 2467 }, "optimeDate" : ISODate("2013-01-18T18:27:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:27:34.384 [conn76] end connection 127.0.0.6:54008 (3 connections now open) m31003| Fri Jan 18 13:27:34.384 [initandlisten] connection accepted from 127.0.0.6:56323 #79 (5 connections now open) m31004| Fri Jan 18 13:27:34.405 [conn92] end connection 127.0.0.3:56359 (4 connections now open) m31004| Fri Jan 18 13:27:34.405 [initandlisten] connection accepted from 127.0.0.3:38815 #96 (5 connections now open) m31004| Fri Jan 18 13:27:35.077 [conn93] end connection 127.0.0.4:45172 (4 connections now open) m31004| Fri Jan 18 13:27:35.077 [initandlisten] connection accepted from 127.0.0.4:38719 #97 (5 connections now open) m31003| Fri Jan 18 13:27:35.182 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 150 more seconds m31003| Fri Jan 18 13:27:35.182 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 149 more seconds m31003| Fri Jan 18 13:27:35.182 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 149 more seconds m31003| Fri Jan 18 13:27:35.182 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 150 more seconds m31003| Fri Jan 18 13:27:35.182 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 149 more seconds m31003| Fri Jan 18 13:27:35.182 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 149 more seconds m31002| Fri Jan 18 13:27:35.431 [conn289] end connection 127.0.0.2:48284 (5 connections now open) m31002| Fri Jan 18 13:27:35.431 [initandlisten] connection accepted from 127.0.0.2:40879 #293 (6 connections now open) m31000| Fri Jan 18 13:27:35.963 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:118286 nreturned:140000 reslen:2380020 118ms m31000| Fri Jan 18 13:27:36.021 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:116788 nreturned:140000 reslen:2380020 116ms m31001| Fri Jan 18 13:27:36.385 [conn382] end connection 127.0.0.6:36370 (4 connections now open) m31001| Fri Jan 18 13:27:36.385 [initandlisten] connection accepted from 127.0.0.6:58290 #386 (5 connections now open) m31002| Fri Jan 18 13:27:36.405 [conn290] end connection 127.0.0.3:42734 (5 connections now open) m31002| Fri Jan 18 13:27:36.405 [initandlisten] connection accepted from 127.0.0.3:38344 #294 (6 connections now open) m31001| Fri Jan 18 13:27:37.289 [conn383] end connection 127.0.0.4:49518 (4 connections now open) m31001| Fri Jan 18 13:27:37.289 [initandlisten] connection accepted from 127.0.0.4:37715 #387 (5 connections now open) m31000| Fri Jan 18 13:27:38.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 732, "optime" : { "t" : 1358533659000, "i" : 3683 }, "optimeDate" : ISODate("2013-01-18T18:27:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 698, "optime" : { "t" : 1358533617000, "i" : 5295 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 698, "optime" : { "t" : 1358533617000, "i" : 5295 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 698, "optime" : { "t" : 1358533659000, "i" : 3167 }, "optimeDate" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 697, "optime" : { "t" : 1358533658000, "i" : 3191 }, "optimeDate" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 732, "optime" : { "t" : 1358533617000, "i" : 5295 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 697, "optime" : { "t" : 1358533617000, "i" : 5295 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:39Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 697, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 697, "optime" : { "t" : 1358533658000, "i" : 3127 }, "optimeDate" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 697, "optime" : { "t" : 1358533659000, "i" : 560 }, "optimeDate" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 697, "optime" : { "t" : 1358533617000, "i" : 5295 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 731, "optime" : { "t" : 1358533617000, "i" : 5295 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 697, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:39Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 697, "optime" : { "t" : 1358533659000, "i" : 555 }, "optimeDate" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:39Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 698, "optime" : { "t" : 1358533617000, "i" : 5295 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:38Z"), "pingMs" : 25 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 698, "optime" : { "t" : 1358533617000, "i" : 5295 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 9 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 730, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 698, "optime" : { "t" : 1358533657000, "i" : 99 }, "optimeDate" : ISODate("2013-01-18T18:27:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 697, "optime" : { "t" : 1358533658000, "i" : 3076 }, "optimeDate" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 697, "optime" : { "t" : 1358533617000, "i" : 5295 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:38Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 697, "optime" : { "t" : 1358533617000, "i" : 5295 }, "optimeDate" : ISODate("2013-01-18T18:26:57Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:39Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 697, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 730, "optime" : { "t" : 1358533659000, "i" : 3167 }, "optimeDate" : ISODate("2013-01-18T18:27:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:27:40.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 145 more seconds m31003| Fri Jan 18 13:27:40.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 144 more seconds m31003| Fri Jan 18 13:27:40.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 144 more seconds m31003| Fri Jan 18 13:27:40.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 145 more seconds m31003| Fri Jan 18 13:27:40.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 144 more seconds m31003| Fri Jan 18 13:27:40.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 144 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 737, "optime" : { "t" : 1358533664000, "i" : 5834 }, "optimeDate" : ISODate("2013-01-18T18:27:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 703, "optime" : { "t" : 1358533622000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 703, "optime" : { "t" : 1358533622000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 703, "optime" : { "t" : 1358533663000, "i" : 3857 }, "optimeDate" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:44Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 702, "optime" : { "t" : 1358533664000, "i" : 3605 }, "optimeDate" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 737, "optime" : { "t" : 1358533622000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 702, "optime" : { "t" : 1358533622000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 702, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 702, "optime" : { "t" : 1358533664000, "i" : 3600 }, "optimeDate" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 702, "optime" : { "t" : 1358533663000, "i" : 370 }, "optimeDate" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 702, "optime" : { "t" : 1358533622000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 736, "optime" : { "t" : 1358533622000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 702, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:43Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 702, "optime" : { "t" : 1358533663000, "i" : 335 }, "optimeDate" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:44Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:44Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 703, "optime" : { "t" : 1358533622000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:44Z"), "pingMs" : 27 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 703, "optime" : { "t" : 1358533622000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 12 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 735, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 703, "optime" : { "t" : 1358533659000, "i" : 3683 }, "optimeDate" : ISODate("2013-01-18T18:27:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:44Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 702, "optime" : { "t" : 1358533664000, "i" : 3452 }, "optimeDate" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 702, "optime" : { "t" : 1358533622000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 702, "optime" : { "t" : 1358533622000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:02Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 702, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 735, "optime" : { "t" : 1358533664000, "i" : 5834 }, "optimeDate" : ISODate("2013-01-18T18:27:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:27:45.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 140 more seconds m31003| Fri Jan 18 13:27:45.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 139 more seconds m31003| Fri Jan 18 13:27:45.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 139 more seconds m31003| Fri Jan 18 13:27:45.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 140 more seconds m31003| Fri Jan 18 13:27:45.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 139 more seconds m31003| Fri Jan 18 13:27:45.183 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 139 more seconds m31001| Fri Jan 18 13:27:45.561 [conn384] end connection 127.0.0.5:56301 (4 connections now open) m31001| Fri Jan 18 13:27:45.618 [initandlisten] connection accepted from 127.0.0.5:60835 #388 (5 connections now open) m31000| Fri Jan 18 13:27:48.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31000| Fri Jan 18 13:27:48.333 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:243615 121ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 742, "optime" : { "t" : 1358533669000, "i" : 8934 }, "optimeDate" : ISODate("2013-01-18T18:27:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 708, "optime" : { "t" : 1358533628000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 708, "optime" : { "t" : 1358533628000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 708, "optime" : { "t" : 1358533669000, "i" : 4833 }, "optimeDate" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 707, "optime" : { "t" : 1358533668000, "i" : 3218 }, "optimeDate" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 742, "optime" : { "t" : 1358533628000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 707, "optime" : { "t" : 1358533627000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:49Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 707, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:49Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 707, "optime" : { "t" : 1358533668000, "i" : 3193 }, "optimeDate" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 707, "optime" : { "t" : 1358533669000, "i" : 32 }, "optimeDate" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 707, "optime" : { "t" : 1358533628000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 741, "optime" : { "t" : 1358533628000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 707, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:49Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 707, "optime" : { "t" : 1358533669000, "i" : 32 }, "optimeDate" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:49Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 708, "optime" : { "t" : 1358533628000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 38 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 708, "optime" : { "t" : 1358533628000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:08Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 24 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 740, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 708, "optime" : { "t" : 1358533669000, "i" : 7143 }, "optimeDate" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 707, "optime" : { "t" : 1358533668000, "i" : 2795 }, "optimeDate" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 707, "optime" : { "t" : 1358533627000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 707, "optime" : { "t" : 1358533627000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:07Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:49Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 707, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:49Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 740, "optime" : { "t" : 1358533669000, "i" : 8934 }, "optimeDate" : ISODate("2013-01-18T18:27:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:27:50.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 135 more seconds m31003| Fri Jan 18 13:27:50.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 134 more seconds m31003| Fri Jan 18 13:27:50.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 134 more seconds m31003| Fri Jan 18 13:27:50.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 135 more seconds m31003| Fri Jan 18 13:27:50.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 134 more seconds m31003| Fri Jan 18 13:27:50.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 134 more seconds m31002| Fri Jan 18 13:27:50.386 [conn291] end connection 127.0.0.6:36928 (5 connections now open) m31002| Fri Jan 18 13:27:50.387 [initandlisten] connection accepted from 127.0.0.6:51132 #295 (6 connections now open) m31003| Fri Jan 18 13:27:50.407 [conn77] end connection 127.0.0.3:44591 (3 connections now open) m31003| Fri Jan 18 13:27:50.407 [initandlisten] connection accepted from 127.0.0.3:33449 #80 (4 connections now open) m31003| Fri Jan 18 13:27:51.079 [conn78] end connection 127.0.0.4:34467 (3 connections now open) m31003| Fri Jan 18 13:27:51.079 [initandlisten] connection accepted from 127.0.0.4:37434 #81 (5 connections now open) m31001| Fri Jan 18 13:27:51.433 [conn385] end connection 127.0.0.2:59505 (4 connections now open) m31001| Fri Jan 18 13:27:51.433 [initandlisten] connection accepted from 127.0.0.2:57832 #389 (5 connections now open) m31000| Fri Jan 18 13:27:52.387 [conn99] end connection 127.0.0.6:42909 (11 connections now open) m31000| Fri Jan 18 13:27:52.387 [initandlisten] connection accepted from 127.0.0.6:46277 #102 (13 connections now open) m31000| Fri Jan 18 13:27:52.408 [conn100] end connection 127.0.0.3:44130 (11 connections now open) m31000| Fri Jan 18 13:27:52.408 [initandlisten] connection accepted from 127.0.0.3:34263 #103 (12 connections now open) m31003| Fri Jan 18 13:27:52.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Fri Jan 18 13:27:53.079 [conn101] end connection 127.0.0.4:33457 (11 connections now open) m31000| Fri Jan 18 13:27:53.079 [initandlisten] connection accepted from 127.0.0.4:45081 #104 (12 connections now open) m31000| Fri Jan 18 13:27:53.386 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:495109 247ms m31000| Fri Jan 18 13:27:54.344 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:1708616 854ms m31004| Fri Jan 18 13:27:54.684 [conn94] end connection 127.0.0.5:48621 (4 connections now open) STATUS FROM : 127.0.0.2:31000 m31004| Fri Jan 18 13:27:55.184 [initandlisten] connection accepted from 127.0.0.5:38412 #98 (5 connections now open) m31003| Fri Jan 18 13:27:55.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 130 more seconds m31003| Fri Jan 18 13:27:55.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 129 more seconds m31003| Fri Jan 18 13:27:55.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 129 more seconds m31003| Fri Jan 18 13:27:55.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 130 more seconds m31003| Fri Jan 18 13:27:55.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 129 more seconds m31003| Fri Jan 18 13:27:55.184 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 129 more seconds { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 748, "optime" : { "t" : 1358533675000, "i" : 5479 }, "optimeDate" : ISODate("2013-01-18T18:27:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 714, "optime" : { "t" : 1358533633000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 714, "optime" : { "t" : 1358533632000, "i" : 5504 }, "optimeDate" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:55Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 714, "optime" : { "t" : 1358533673000, "i" : 1227 }, "optimeDate" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:54Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 713, "optime" : { "t" : 1358533674000, "i" : 1338 }, "optimeDate" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 748, "optime" : { "t" : 1358533633000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 713, "optime" : { "t" : 1358533633000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:55Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 713, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 713, "optime" : { "t" : 1358533673000, "i" : 3392 }, "optimeDate" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 713, "optime" : { "t" : 1358533675000, "i" : 246 }, "optimeDate" : ISODate("2013-01-18T18:27:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 713, "optime" : { "t" : 1358533633000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 747, "optime" : { "t" : 1358533633000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 713, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 713, "optime" : { "t" : 1358533675000, "i" : 207 }, "optimeDate" : ISODate("2013-01-18T18:27:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:55Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 714, "optime" : { "t" : 1358533633000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:54Z"), "pingMs" : 45 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 714, "optime" : { "t" : 1358533632000, "i" : 5504 }, "optimeDate" : ISODate("2013-01-18T18:27:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:55Z"), "pingMs" : 39 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 746, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 714, "optime" : { "t" : 1358533675000, "i" : 2879 }, "optimeDate" : ISODate("2013-01-18T18:27:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:27:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 713, "optime" : { "t" : 1358533674000, "i" : 837 }, "optimeDate" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 713, "optime" : { "t" : 1358533633000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 713, "optime" : { "t" : 1358533633000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:13Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:55Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 713, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:55Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 746, "optime" : { "t" : 1358533675000, "i" : 5479 }, "optimeDate" : ISODate("2013-01-18T18:27:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:27:59.515 [conn292] end connection 127.0.0.5:42430 (5 connections now open) m31002| Fri Jan 18 13:27:59.590 [initandlisten] connection accepted from 127.0.0.5:42684 #296 (6 connections now open) m31003| Fri Jan 18 13:28:00.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 125 more seconds m31003| Fri Jan 18 13:28:00.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 124 more seconds m31003| Fri Jan 18 13:28:00.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 124 more seconds m31003| Fri Jan 18 13:28:00.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 125 more seconds m31003| Fri Jan 18 13:28:00.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 124 more seconds m31003| Fri Jan 18 13:28:00.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 124 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 753, "optime" : { "t" : 1358533680000, "i" : 5796 }, "optimeDate" : ISODate("2013-01-18T18:28:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 719, "optime" : { "t" : 1358533638000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 719, "optime" : { "t" : 1358533638000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 719, "optime" : { "t" : 1358533679000, "i" : 5441 }, "optimeDate" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 718, "optime" : { "t" : 1358533680000, "i" : 2499 }, "optimeDate" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 753, "optime" : { "t" : 1358533639000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 718, "optime" : { "t" : 1358533638000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 718, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 718, "optime" : { "t" : 1358533680000, "i" : 2499 }, "optimeDate" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 718, "optime" : { "t" : 1358533679000, "i" : 1972 }, "optimeDate" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 718, "optime" : { "t" : 1358533638000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 752, "optime" : { "t" : 1358533638000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:18Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 718, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:27:59Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 718, "optime" : { "t" : 1358533679000, "i" : 952 }, "optimeDate" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:00Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 719, "optime" : { "t" : 1358533639000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:00Z"), "pingMs" : 48 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 719, "optime" : { "t" : 1358533638000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 42 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 751, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 719, "optime" : { "t" : 1358533680000, "i" : 1232 }, "optimeDate" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 718, "optime" : { "t" : 1358533680000, "i" : 2499 }, "optimeDate" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 718, "optime" : { "t" : 1358533639000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 718, "optime" : { "t" : 1358533638000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 718, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 751, "optime" : { "t" : 1358533680000, "i" : 5796 }, "optimeDate" : ISODate("2013-01-18T18:28:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:28:01.142 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.4, filling with zeroes... m31001| Fri Jan 18 13:28:02.033 [FileAllocator] allocating new datafile /data/db/testReplSet-1/foo.4, filling with zeroes... m31003| Fri Jan 18 13:28:02.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Fri Jan 18 13:28:03.183 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.4, size: 256MB, took 2.04 secs m31004| Fri Jan 18 13:28:03.435 [conn95] end connection 127.0.0.2:48541 (4 connections now open) m31004| Fri Jan 18 13:28:03.435 [initandlisten] connection accepted from 127.0.0.2:47020 #99 (5 connections now open) m31001| Fri Jan 18 13:28:04.110 [FileAllocator] done allocating datafile /data/db/testReplSet-1/foo.4, size: 256MB, took 2.077 secs m31003| Fri Jan 18 13:28:04.389 [conn79] end connection 127.0.0.6:56323 (3 connections now open) m31003| Fri Jan 18 13:28:04.389 [initandlisten] connection accepted from 127.0.0.6:35625 #82 (4 connections now open) m31004| Fri Jan 18 13:28:04.409 [conn96] end connection 127.0.0.3:38815 (4 connections now open) m31004| Fri Jan 18 13:28:04.409 [initandlisten] connection accepted from 127.0.0.3:47628 #100 (6 connections now open) m31004| Fri Jan 18 13:28:05.081 [conn97] end connection 127.0.0.4:38719 (4 connections now open) m31004| Fri Jan 18 13:28:05.081 [initandlisten] connection accepted from 127.0.0.4:60655 #101 (5 connections now open) m31003| Fri Jan 18 13:28:05.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 120 more seconds m31003| Fri Jan 18 13:28:05.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 119 more seconds m31003| Fri Jan 18 13:28:05.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 119 more seconds m31003| Fri Jan 18 13:28:05.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 120 more seconds m31003| Fri Jan 18 13:28:05.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 119 more seconds m31003| Fri Jan 18 13:28:05.185 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 119 more seconds m31002| Fri Jan 18 13:28:05.435 [conn293] end connection 127.0.0.2:40879 (5 connections now open) m31002| Fri Jan 18 13:28:05.435 [initandlisten] connection accepted from 127.0.0.2:59009 #297 (6 connections now open) m31003| Fri Jan 18 13:28:05.933 [rsSync] replSet still syncing, not yet to minValid optime 50f991f5:10f5 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 759, "optime" : { "t" : 1358533686000, "i" : 428 }, "optimeDate" : ISODate("2013-01-18T18:28:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 725, "optime" : { "t" : 1358533645000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 725, "optime" : { "t" : 1358533645000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 725, "optime" : { "t" : 1358533685000, "i" : 3212 }, "optimeDate" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 724, "optime" : { "t" : 1358533684000, "i" : 3648 }, "optimeDate" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 759, "optime" : { "t" : 1358533645000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:25Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 724, "optime" : { "t" : 1358533644000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:05Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 724, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:06Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 724, "optime" : { "t" : 1358533684000, "i" : 3639 }, "optimeDate" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 724, "optime" : { "t" : 1358533685000, "i" : 601 }, "optimeDate" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 724, "optime" : { "t" : 1358533641000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 758, "optime" : { "t" : 1358533645000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:25Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 724, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:05Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 724, "optime" : { "t" : 1358533685000, "i" : 594 }, "optimeDate" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:06Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 725, "optime" : { "t" : 1358533645000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 725, "optime" : { "t" : 1358533645000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 29 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 757, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 725, "optime" : { "t" : 1358533685000, "i" : 1557 }, "optimeDate" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 724, "optime" : { "t" : 1358533684000, "i" : 3337 }, "optimeDate" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 724, "optime" : { "t" : 1358533641000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:04Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 724, "optime" : { "t" : 1358533644000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:05Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 724, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:05Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 757, "optime" : { "t" : 1358533686000, "i" : 428 }, "optimeDate" : ISODate("2013-01-18T18:28:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31001| Fri Jan 18 13:28:06.389 [conn386] end connection 127.0.0.6:58290 (4 connections now open) m31001| Fri Jan 18 13:28:06.389 [initandlisten] connection accepted from 127.0.0.6:57413 #390 (5 connections now open) m31002| Fri Jan 18 13:28:06.410 [conn294] end connection 127.0.0.3:38344 (5 connections now open) m31002| Fri Jan 18 13:28:06.410 [initandlisten] connection accepted from 127.0.0.3:49367 #298 (6 connections now open) m31001| Fri Jan 18 13:28:07.293 [conn387] end connection 127.0.0.4:37715 (4 connections now open) m31001| Fri Jan 18 13:28:07.293 [initandlisten] connection accepted from 127.0.0.4:46861 #391 (5 connections now open) m31000| Fri Jan 18 13:28:10.040 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:1917687 958ms m31003| Fri Jan 18 13:28:10.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 115 more seconds m31003| Fri Jan 18 13:28:10.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 114 more seconds m31003| Fri Jan 18 13:28:10.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 114 more seconds m31003| Fri Jan 18 13:28:10.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 115 more seconds m31003| Fri Jan 18 13:28:10.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 114 more seconds m31003| Fri Jan 18 13:28:10.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 114 more seconds m31000| Fri Jan 18 13:28:10.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 764, "optime" : { "t" : 1358533691000, "i" : 2367 }, "optimeDate" : ISODate("2013-01-18T18:28:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 730, "optime" : { "t" : 1358533647000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 730, "optime" : { "t" : 1358533647000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:11Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 730, "optime" : { "t" : 1358533689000, "i" : 3 }, "optimeDate" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:10Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 729, "optime" : { "t" : 1358533690000, "i" : 7400 }, "optimeDate" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 764, "optime" : { "t" : 1358533650000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:30Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 729, "optime" : { "t" : 1358533647000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 729, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 729, "optime" : { "t" : 1358533690000, "i" : 7281 }, "optimeDate" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 729, "optime" : { "t" : 1358533691000, "i" : 2130 }, "optimeDate" : ISODate("2013-01-18T18:28:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 729, "optime" : { "t" : 1358533647000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 763, "optime" : { "t" : 1358533650000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:30Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 729, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 729, "optime" : { "t" : 1358533691000, "i" : 2107 }, "optimeDate" : ISODate("2013-01-18T18:28:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:11Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 730, "optime" : { "t" : 1358533647000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:10Z"), "pingMs" : 37 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 730, "optime" : { "t" : 1358533647000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:11Z"), "pingMs" : 24 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 762, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 730, "optime" : { "t" : 1358533690000, "i" : 2822 }, "optimeDate" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 729, "optime" : { "t" : 1358533690000, "i" : 6817 }, "optimeDate" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 729, "optime" : { "t" : 1358533647000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 729, "optime" : { "t" : 1358533647000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:11Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 729, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 762, "optime" : { "t" : 1358533691000, "i" : 2367 }, "optimeDate" : ISODate("2013-01-18T18:28:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:28:14.401 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:480184 240ms m31003| Fri Jan 18 13:28:15.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 110 more seconds m31003| Fri Jan 18 13:28:15.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 109 more seconds m31003| Fri Jan 18 13:28:15.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 109 more seconds m31003| Fri Jan 18 13:28:15.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 110 more seconds m31003| Fri Jan 18 13:28:15.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 109 more seconds m31003| Fri Jan 18 13:28:15.186 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 109 more seconds STATUS FROM : 127.0.0.2:31000 m31001| Fri Jan 18 13:28:16.214 [conn388] end connection 127.0.0.5:60835 (4 connections now open) m31001| Fri Jan 18 13:28:16.214 [initandlisten] connection accepted from 127.0.0.5:53379 #392 (6 connections now open) m31000| Fri Jan 18 13:28:18.149 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:7288240 3644ms m31000| Fri Jan 18 13:28:18.150 [conn24] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185723707855858 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:54 nreturned:5 reslen:535 3647ms m31000| Fri Jan 18 13:28:18.150 [conn26] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185723782124859 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:47 nreturned:5 reslen:535 3647ms m31000| Fri Jan 18 13:28:18.150 [conn23] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532932000|1 } } cursorid:185725031169493 ntoreturn:0 keyUpdates:0 numYields: 1 locks(micros) r:99 nreturned:6 reslen:638 3647ms m31000| Fri Jan 18 13:28:18.366 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:217657 108ms m31000| Fri Jan 18 13:28:18.532 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 numYields: 1087 locks(micros) r:337851 nreturned:145385 reslen:2471565 289ms m31000| Fri Jan 18 13:28:18.585 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 numYields: 1088 locks(micros) r:287038 nreturned:145401 reslen:2471837 333ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 772, "optime" : { "t" : 1358533699000, "i" : 8251 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 738, "optime" : { "t" : 1358533658000, "i" : 7148 }, "optimeDate" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 738, "optime" : { "t" : 1358533658000, "i" : 7148 }, "optimeDate" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 738, "optime" : { "t" : 1358533699000, "i" : 8068 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 737, "optime" : { "t" : 1358533698000, "i" : 1946 }, "optimeDate" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:19Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 772, "optime" : { "t" : 1358533658000, "i" : 7148 }, "optimeDate" : ISODate("2013-01-18T18:27:38Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 737, "optime" : { "t" : 1358533657000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:19Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 737, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 737, "optime" : { "t" : 1358533698000, "i" : 1940 }, "optimeDate" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 737, "optime" : { "t" : 1358533699000, "i" : 2087 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:19Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 737, "optime" : { "t" : 1358533658000, "i" : 7148 }, "optimeDate" : ISODate("2013-01-18T18:27:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 771, "optime" : { "t" : 1358533658000, "i" : 7148 }, "optimeDate" : ISODate("2013-01-18T18:27:38Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 737, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 737, "optime" : { "t" : 1358533699000, "i" : 344 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:19Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 738, "optime" : { "t" : 1358533657000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:18Z"), "pingMs" : 27 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 738, "optime" : { "t" : 1358533654000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:19Z"), "pingMs" : 17 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 770, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 738, "optime" : { "t" : 1358533694000, "i" : 371 }, "optimeDate" : ISODate("2013-01-18T18:28:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 737, "optime" : { "t" : 1358533698000, "i" : 1914 }, "optimeDate" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:19Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 737, "optime" : { "t" : 1358533657000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:18Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 737, "optime" : { "t" : 1358533657000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:19Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 737, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 770, "optime" : { "t" : 1358533699000, "i" : 8251 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:28:20.187 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 105 more seconds m31003| Fri Jan 18 13:28:20.187 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 104 more seconds m31003| Fri Jan 18 13:28:20.187 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 104 more seconds m31003| Fri Jan 18 13:28:20.187 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 105 more seconds m31003| Fri Jan 18 13:28:20.187 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 104 more seconds m31003| Fri Jan 18 13:28:20.187 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 104 more seconds m31000| Fri Jan 18 13:28:20.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31002| Fri Jan 18 13:28:20.391 [conn295] end connection 127.0.0.6:51132 (5 connections now open) m31002| Fri Jan 18 13:28:20.391 [initandlisten] connection accepted from 127.0.0.6:48948 #299 (6 connections now open) m31003| Fri Jan 18 13:28:20.411 [conn80] end connection 127.0.0.3:33449 (3 connections now open) m31003| Fri Jan 18 13:28:20.411 [initandlisten] connection accepted from 127.0.0.3:52845 #83 (4 connections now open) m31003| Fri Jan 18 13:28:21.083 [conn81] end connection 127.0.0.4:37434 (3 connections now open) m31003| Fri Jan 18 13:28:21.083 [initandlisten] connection accepted from 127.0.0.4:36033 #84 (4 connections now open) m31001| Fri Jan 18 13:28:21.437 [conn389] end connection 127.0.0.2:57832 (4 connections now open) m31001| Fri Jan 18 13:28:21.437 [initandlisten] connection accepted from 127.0.0.2:33340 #393 (5 connections now open) m31000| Fri Jan 18 13:28:22.391 [conn102] end connection 127.0.0.6:46277 (11 connections now open) m31000| Fri Jan 18 13:28:22.391 [initandlisten] connection accepted from 127.0.0.6:33230 #105 (12 connections now open) m31000| Fri Jan 18 13:28:22.412 [conn103] end connection 127.0.0.3:34263 (11 connections now open) m31000| Fri Jan 18 13:28:22.412 [initandlisten] connection accepted from 127.0.0.3:42855 #106 (13 connections now open) m31000| Fri Jan 18 13:28:23.083 [conn104] end connection 127.0.0.4:45081 (11 connections now open) m31000| Fri Jan 18 13:28:23.083 [initandlisten] connection accepted from 127.0.0.4:38251 #107 (12 connections now open) m31003| Fri Jan 18 13:28:24.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 777, "optime" : { "t" : 1358533704000, "i" : 6188 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 743, "optime" : { "t" : 1358533662000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:24Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 743, "optime" : { "t" : 1358533663000, "i" : 4973 }, "optimeDate" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 743, "optime" : { "t" : 1358533703000, "i" : 4776 }, "optimeDate" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:24Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 742, "optime" : { "t" : 1358533704000, "i" : 3393 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 777, "optime" : { "t" : 1358533664000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:44Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 742, "optime" : { "t" : 1358533663000, "i" : 4973 }, "optimeDate" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 742, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 742, "optime" : { "t" : 1358533704000, "i" : 3384 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 742, "optime" : { "t" : 1358533703000, "i" : 1517 }, "optimeDate" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:23Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 742, "optime" : { "t" : 1358533662000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:24Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 776, "optime" : { "t" : 1358533664000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:44Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 742, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:23Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 742, "optime" : { "t" : 1358533703000, "i" : 1503 }, "optimeDate" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:24Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:24Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 743, "optime" : { "t" : 1358533663000, "i" : 4973 }, "optimeDate" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:24Z"), "pingMs" : 29 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 743, "optime" : { "t" : 1358533663000, "i" : 4973 }, "optimeDate" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 18 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 775, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 743, "optime" : { "t" : 1358533702000, "i" : 1413 }, "optimeDate" : ISODate("2013-01-18T18:28:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 742, "optime" : { "t" : 1358533704000, "i" : 3393 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 742, "optime" : { "t" : 1358533663000, "i" : 4973 }, "optimeDate" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:24Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 742, "optime" : { "t" : 1358533663000, "i" : 4973 }, "optimeDate" : ISODate("2013-01-18T18:27:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 742, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 775, "optime" : { "t" : 1358533704000, "i" : 6188 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:28:25.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 100 more seconds m31003| Fri Jan 18 13:28:25.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 99 more seconds m31003| Fri Jan 18 13:28:25.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 99 more seconds m31003| Fri Jan 18 13:28:25.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 100 more seconds m31003| Fri Jan 18 13:28:25.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 99 more seconds m31003| Fri Jan 18 13:28:25.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 99 more seconds m31000| Fri Jan 18 13:28:28.121 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:265869 132ms m31000| Fri Jan 18 13:28:28.734 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:314811 157ms m31000| Fri Jan 18 13:28:28.976 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:274877 137ms STATUS FROM : 127.0.0.2:31000 m31002| Fri Jan 18 13:28:29.901 [conn296] end connection 127.0.0.5:42684 (5 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 782, "optime" : { "t" : 1358533709000, "i" : 9170 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 748, "optime" : { "t" : 1358533668000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 748, "optime" : { "t" : 1358533669000, "i" : 6917 }, "optimeDate" : ISODate("2013-01-18T18:27:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:27:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 748, "optime" : { "t" : 1358533709000, "i" : 5391 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 747, "optime" : { "t" : 1358533708000, "i" : 4520 }, "optimeDate" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 782, "optime" : { "t" : 1358533669000, "i" : 6917 }, "optimeDate" : ISODate("2013-01-18T18:27:49Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 747, "optime" : { "t" : 1358533667000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:29Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 747, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:28Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 747, "optime" : { "t" : 1358533708000, "i" : 4349 }, "optimeDate" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 747, "optime" : { "t" : 1358533709000, "i" : 1865 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 747, "optime" : { "t" : 1358533668000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 781, "optime" : { "t" : 1358533669000, "i" : 6917 }, "optimeDate" : ISODate("2013-01-18T18:27:49Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 747, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 747, "optime" : { "t" : 1358533709000, "i" : 1728 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:29Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 748, "optime" : { "t" : 1358533667000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 32 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 748, "optime" : { "t" : 1358533667000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:29Z"), "pingMs" : 11 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 780, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 748, "optime" : { "t" : 1358533704000, "i" : 6188 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 747, "optime" : { "t" : 1358533708000, "i" : 4107 }, "optimeDate" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 747, "optime" : { "t" : 1358533667000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:28Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 747, "optime" : { "t" : 1358533667000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:29Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 747, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 780, "optime" : { "t" : 1358533709000, "i" : 9170 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:28:29.999 [initandlisten] connection accepted from 127.0.0.5:50300 #300 (6 connections now open) m31003| Fri Jan 18 13:28:30.027 [rsMgr] replSet I don't see a primary and I can't elect myself m31003| Fri Jan 18 13:28:30.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 95 more seconds m31003| Fri Jan 18 13:28:30.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 94 more seconds m31003| Fri Jan 18 13:28:30.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 94 more seconds m31003| Fri Jan 18 13:28:30.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 95 more seconds m31003| Fri Jan 18 13:28:30.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 94 more seconds m31003| Fri Jan 18 13:28:30.188 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 94 more seconds m31004| Fri Jan 18 13:28:32.689 [conn98] end connection 127.0.0.5:38412 (4 connections now open) m31004| Fri Jan 18 13:28:33.189 [initandlisten] connection accepted from 127.0.0.5:33663 #102 (5 connections now open) m31004| Fri Jan 18 13:28:33.439 [conn99] end connection 127.0.0.2:47020 (4 connections now open) m31004| Fri Jan 18 13:28:33.439 [initandlisten] connection accepted from 127.0.0.2:54419 #103 (5 connections now open) m31003| Fri Jan 18 13:28:34.393 [conn82] end connection 127.0.0.6:35625 (3 connections now open) m31003| Fri Jan 18 13:28:34.393 [initandlisten] connection accepted from 127.0.0.6:38291 #85 (4 connections now open) m31004| Fri Jan 18 13:28:34.414 [conn100] end connection 127.0.0.3:47628 (4 connections now open) m31004| Fri Jan 18 13:28:34.414 [initandlisten] connection accepted from 127.0.0.3:43602 #104 (5 connections now open) m31003| Fri Jan 18 13:28:34.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31004| Fri Jan 18 13:28:35.085 [conn101] end connection 127.0.0.4:60655 (4 connections now open) m31004| Fri Jan 18 13:28:35.085 [initandlisten] connection accepted from 127.0.0.4:47604 #105 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31003| Fri Jan 18 13:28:35.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 90 more seconds m31003| Fri Jan 18 13:28:35.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 89 more seconds m31003| Fri Jan 18 13:28:35.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 89 more seconds m31003| Fri Jan 18 13:28:35.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 90 more seconds m31003| Fri Jan 18 13:28:35.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 89 more seconds m31003| Fri Jan 18 13:28:35.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 89 more seconds { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 788, "optime" : { "t" : 1358533715000, "i" : 2660 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 754, "optime" : { "t" : 1358533672000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:34Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 754, "optime" : { "t" : 1358533672000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:35Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 754, "optime" : { "t" : 1358533713000, "i" : 3873 }, "optimeDate" : ISODate("2013-01-18T18:28:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 753, "optime" : { "t" : 1358533714000, "i" : 3603 }, "optimeDate" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 788, "optime" : { "t" : 1358533674000, "i" : 2066 }, "optimeDate" : ISODate("2013-01-18T18:27:54Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 753, "optime" : { "t" : 1358533673000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 753, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:34Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 753, "optime" : { "t" : 1358533714000, "i" : 3024 }, "optimeDate" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 753, "optime" : { "t" : 1358533715000, "i" : 1596 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 753, "optime" : { "t" : 1358533672000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:34Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 787, "optime" : { "t" : 1358533674000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:27:54Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 753, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 753, "optime" : { "t" : 1358533715000, "i" : 1538 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:35Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 754, "optime" : { "t" : 1358533673000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 16 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 754, "optime" : { "t" : 1358533673000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:35Z"), "pingMs" : 26 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 786, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 754, "optime" : { "t" : 1358533713000, "i" : 1562 }, "optimeDate" : ISODate("2013-01-18T18:28:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 753, "optime" : { "t" : 1358533714000, "i" : 3131 }, "optimeDate" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 753, "optime" : { "t" : 1358533673000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:34Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 753, "optime" : { "t" : 1358533673000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:53Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:35Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 753, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 786, "optime" : { "t" : 1358533715000, "i" : 2560 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31002| Fri Jan 18 13:28:35.440 [conn297] end connection 127.0.0.2:59009 (5 connections now open) m31002| Fri Jan 18 13:28:35.440 [initandlisten] connection accepted from 127.0.0.2:58567 #301 (6 connections now open) m31001| Fri Jan 18 13:28:36.393 [conn390] end connection 127.0.0.6:57413 (4 connections now open) m31001| Fri Jan 18 13:28:36.393 [initandlisten] connection accepted from 127.0.0.6:45310 #394 (5 connections now open) m31002| Fri Jan 18 13:28:36.414 [conn298] end connection 127.0.0.3:49367 (5 connections now open) m31002| Fri Jan 18 13:28:36.414 [initandlisten] connection accepted from 127.0.0.3:35803 #302 (6 connections now open) m31001| Fri Jan 18 13:28:37.298 [conn391] end connection 127.0.0.4:46861 (4 connections now open) m31001| Fri Jan 18 13:28:37.298 [initandlisten] connection accepted from 127.0.0.4:37529 #395 (5 connections now open) m31003| Fri Jan 18 13:28:40.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 85 more seconds m31003| Fri Jan 18 13:28:40.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 84 more seconds m31003| Fri Jan 18 13:28:40.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 84 more seconds m31003| Fri Jan 18 13:28:40.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 85 more seconds m31003| Fri Jan 18 13:28:40.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 84 more seconds m31003| Fri Jan 18 13:28:40.189 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 84 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 793, "optime" : { "t" : 1358533720000, "i" : 4173 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 759, "optime" : { "t" : 1358533678000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 759, "optime" : { "t" : 1358533679000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 759, "optime" : { "t" : 1358533719000, "i" : 3818 }, "optimeDate" : ISODate("2013-01-18T18:28:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 758, "optime" : { "t" : 1358533720000, "i" : 3796 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 793, "optime" : { "t" : 1358533680000, "i" : 4763 }, "optimeDate" : ISODate("2013-01-18T18:28:00Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 758, "optime" : { "t" : 1358533679000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 758, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:40Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 758, "optime" : { "t" : 1358533720000, "i" : 3795 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 758, "optime" : { "t" : 1358533719000, "i" : 389 }, "optimeDate" : ISODate("2013-01-18T18:28:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 758, "optime" : { "t" : 1358533678000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:27:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 792, "optime" : { "t" : 1358533679000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:27:59Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 758, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:40Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 758, "optime" : { "t" : 1358533719000, "i" : 389 }, "optimeDate" : ISODate("2013-01-18T18:28:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:40Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 759, "optime" : { "t" : 1358533680000, "i" : 4763 }, "optimeDate" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 7 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 759, "optime" : { "t" : 1358533679000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 25 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 791, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 759, "optime" : { "t" : 1358533715000, "i" : 2660 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 758, "optime" : { "t" : 1358533720000, "i" : 3651 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 758, "optime" : { "t" : 1358533680000, "i" : 4763 }, "optimeDate" : ISODate("2013-01-18T18:28:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:40Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 758, "optime" : { "t" : 1358533679000, "i" : 10003 }, "optimeDate" : ISODate("2013-01-18T18:27:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 758, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 791, "optime" : { "t" : 1358533720000, "i" : 3795 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:28:42.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:28:45.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 80 more seconds m31003| Fri Jan 18 13:28:45.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 79 more seconds m31003| Fri Jan 18 13:28:45.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 79 more seconds m31003| Fri Jan 18 13:28:45.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 80 more seconds m31003| Fri Jan 18 13:28:45.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 79 more seconds m31003| Fri Jan 18 13:28:45.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 79 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 798, "optime" : { "t" : 1358533725000, "i" : 6521 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 764, "optime" : { "t" : 1358533684000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 764, "optime" : { "t" : 1358533685000, "i" : 6298 }, "optimeDate" : ISODate("2013-01-18T18:28:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 764, "optime" : { "t" : 1358533725000, "i" : 3667 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 763, "optime" : { "t" : 1358533724000, "i" : 3474 }, "optimeDate" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 798, "optime" : { "t" : 1358533685000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 763, "optime" : { "t" : 1358533683000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:45Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 763, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:44Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 763, "optime" : { "t" : 1358533724000, "i" : 3474 }, "optimeDate" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 763, "optime" : { "t" : 1358533725000, "i" : 379 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 763, "optime" : { "t" : 1358533684000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 797, "optime" : { "t" : 1358533685000, "i" : 6298 }, "optimeDate" : ISODate("2013-01-18T18:28:05Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 763, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 763, "optime" : { "t" : 1358533725000, "i" : 379 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:45Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 764, "optime" : { "t" : 1358533684000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 4 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 764, "optime" : { "t" : 1358533683000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:45Z"), "pingMs" : 16 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 796, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 764, "optime" : { "t" : 1358533725000, "i" : 5748 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 763, "optime" : { "t" : 1358533724000, "i" : 3376 }, "optimeDate" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 763, "optime" : { "t" : 1358533684000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:44Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 763, "optime" : { "t" : 1358533683000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:45Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 763, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:45Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 796, "optime" : { "t" : 1358533725000, "i" : 6521 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31001| Fri Jan 18 13:28:46.562 [conn392] end connection 127.0.0.5:53379 (4 connections now open) m31001| Fri Jan 18 13:28:46.562 [initandlisten] connection accepted from 127.0.0.5:44740 #396 (6 connections now open) m31003| Fri Jan 18 13:28:50.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 75 more seconds m31003| Fri Jan 18 13:28:50.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 74 more seconds m31003| Fri Jan 18 13:28:50.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 74 more seconds m31003| Fri Jan 18 13:28:50.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 75 more seconds m31003| Fri Jan 18 13:28:50.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 74 more seconds m31003| Fri Jan 18 13:28:50.190 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 74 more seconds m31002| Fri Jan 18 13:28:50.395 [conn299] end connection 127.0.0.6:48948 (5 connections now open) m31002| Fri Jan 18 13:28:50.395 [initandlisten] connection accepted from 127.0.0.6:44375 #303 (6 connections now open) m31003| Fri Jan 18 13:28:50.416 [conn83] end connection 127.0.0.3:52845 (3 connections now open) m31003| Fri Jan 18 13:28:50.416 [initandlisten] connection accepted from 127.0.0.3:60091 #86 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 804, "optime" : { "t" : 1358533731000, "i" : 389 }, "optimeDate" : ISODate("2013-01-18T18:28:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 770, "optime" : { "t" : 1358533686000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:50Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 770, "optime" : { "t" : 1358533686000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 770, "optime" : { "t" : 1358533729000, "i" : 4339 }, "optimeDate" : ISODate("2013-01-18T18:28:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:50Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 769, "optime" : { "t" : 1358533730000, "i" : 3541 }, "optimeDate" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 804, "optime" : { "t" : 1358533690000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:10Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 769, "optime" : { "t" : 1358533689000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 769, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:50Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 769, "optime" : { "t" : 1358533730000, "i" : 3541 }, "optimeDate" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 769, "optime" : { "t" : 1358533729000, "i" : 1276 }, "optimeDate" : ISODate("2013-01-18T18:28:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 769, "optime" : { "t" : 1358533686000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:50Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 803, "optime" : { "t" : 1358533690000, "i" : 15004 }, "optimeDate" : ISODate("2013-01-18T18:28:10Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 769, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:50Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 769, "optime" : { "t" : 1358533729000, "i" : 1261 }, "optimeDate" : ISODate("2013-01-18T18:28:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:50Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:51Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 770, "optime" : { "t" : 1358533689000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 25 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 770, "optime" : { "t" : 1358533689000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 21 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 802, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 770, "optime" : { "t" : 1358533730000, "i" : 6156 }, "optimeDate" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 769, "optime" : { "t" : 1358533730000, "i" : 3528 }, "optimeDate" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 769, "optime" : { "t" : 1358533689000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:50Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 769, "optime" : { "t" : 1358533689000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 769, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:50Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 802, "optime" : { "t" : 1358533731000, "i" : 389 }, "optimeDate" : ISODate("2013-01-18T18:28:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:28:51.088 [conn84] end connection 127.0.0.4:36033 (3 connections now open) m31003| Fri Jan 18 13:28:51.088 [initandlisten] connection accepted from 127.0.0.4:39688 #87 (4 connections now open) m31001| Fri Jan 18 13:28:51.442 [conn393] end connection 127.0.0.2:33340 (4 connections now open) m31001| Fri Jan 18 13:28:51.442 [initandlisten] connection accepted from 127.0.0.2:44416 #397 (6 connections now open) m31000| Fri Jan 18 13:28:52.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31000| Fri Jan 18 13:28:52.396 [conn105] end connection 127.0.0.6:33230 (11 connections now open) m31000| Fri Jan 18 13:28:52.396 [initandlisten] connection accepted from 127.0.0.6:59669 #108 (12 connections now open) m31000| Fri Jan 18 13:28:52.416 [conn106] end connection 127.0.0.3:42855 (11 connections now open) m31000| Fri Jan 18 13:28:52.416 [initandlisten] connection accepted from 127.0.0.3:44016 #109 (12 connections now open) m31000| Fri Jan 18 13:28:53.088 [conn107] end connection 127.0.0.4:38251 (11 connections now open) m31000| Fri Jan 18 13:28:53.088 [initandlisten] connection accepted from 127.0.0.4:39487 #110 (12 connections now open) m31003| Fri Jan 18 13:28:55.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 70 more seconds m31003| Fri Jan 18 13:28:55.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 69 more seconds m31003| Fri Jan 18 13:28:55.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 69 more seconds m31003| Fri Jan 18 13:28:55.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 70 more seconds m31003| Fri Jan 18 13:28:55.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 69 more seconds m31003| Fri Jan 18 13:28:55.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 69 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 809, "optime" : { "t" : 1358533736000, "i" : 3249 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 775, "optime" : { "t" : 1358533694000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 775, "optime" : { "t" : 1358533694000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 775, "optime" : { "t" : 1358533735000, "i" : 3566 }, "optimeDate" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 774, "optime" : { "t" : 1358533734000, "i" : 3151 }, "optimeDate" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 809, "optime" : { "t" : 1358533694000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:14Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 774, "optime" : { "t" : 1358533691000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:55Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 774, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:54Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 774, "optime" : { "t" : 1358533734000, "i" : 3132 }, "optimeDate" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 774, "optime" : { "t" : 1358533735000, "i" : 922 }, "optimeDate" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 774, "optime" : { "t" : 1358533694000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 808, "optime" : { "t" : 1358533694000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:14Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 774, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 774, "optime" : { "t" : 1358533735000, "i" : 533 }, "optimeDate" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:56Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 775, "optime" : { "t" : 1358533694000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 22 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 775, "optime" : { "t" : 1358533691000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:55Z"), "pingMs" : 12 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 807, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 775, "optime" : { "t" : 1358533735000, "i" : 5850 }, "optimeDate" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:28:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 774, "optime" : { "t" : 1358533734000, "i" : 3005 }, "optimeDate" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 774, "optime" : { "t" : 1358533690000, "i" : 15004 }, "optimeDate" : ISODate("2013-01-18T18:28:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 774, "optime" : { "t" : 1358533691000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:55Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 774, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:28:55Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 807, "optime" : { "t" : 1358533736000, "i" : 3130 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:28:56.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Fri Jan 18 13:28:58.476 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:133286 nreturned:154599 reslen:2628203 133ms m31000| Fri Jan 18 13:28:58.989 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:129031 nreturned:154615 reslen:2628475 129ms m31003| Fri Jan 18 13:29:00.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 65 more seconds m31003| Fri Jan 18 13:29:00.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 64 more seconds m31003| Fri Jan 18 13:29:00.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 64 more seconds m31003| Fri Jan 18 13:29:00.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 65 more seconds m31003| Fri Jan 18 13:29:00.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 64 more seconds m31003| Fri Jan 18 13:29:00.191 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 64 more seconds m31002| Fri Jan 18 13:29:00.371 [conn300] end connection 127.0.0.5:50300 (5 connections now open) m31002| Fri Jan 18 13:29:00.468 [initandlisten] connection accepted from 127.0.0.5:49286 #304 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 814, "optime" : { "t" : 1358533741000, "i" : 6439 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 780, "optime" : { "t" : 1358533699000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 780, "optime" : { "t" : 1358533699000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 780, "optime" : { "t" : 1358533741000, "i" : 3510 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 779, "optime" : { "t" : 1358533740000, "i" : 3896 }, "optimeDate" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 814, "optime" : { "t" : 1358533699000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 779, "optime" : { "t" : 1358533699000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:01Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 779, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 779, "optime" : { "t" : 1358533740000, "i" : 3881 }, "optimeDate" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 779, "optime" : { "t" : 1358533741000, "i" : 843 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 779, "optime" : { "t" : 1358533699000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 813, "optime" : { "t" : 1358533699000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 779, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 779, "optime" : { "t" : 1358533741000, "i" : 834 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:01Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 780, "optime" : { "t" : 1358533699000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 11 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 780, "optime" : { "t" : 1358533699000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:01Z"), "pingMs" : 31 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 812, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 780, "optime" : { "t" : 1358533740000, "i" : 6173 }, "optimeDate" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 779, "optime" : { "t" : 1358533740000, "i" : 3786 }, "optimeDate" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 779, "optime" : { "t" : 1358533699000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 779, "optime" : { "t" : 1358533699000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:01Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 779, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 812, "optime" : { "t" : 1358533741000, "i" : 6439 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:29:03.444 [conn103] end connection 127.0.0.2:54419 (4 connections now open) m31004| Fri Jan 18 13:29:03.444 [initandlisten] connection accepted from 127.0.0.2:43392 #106 (5 connections now open) m31003| Fri Jan 18 13:29:04.397 [conn85] end connection 127.0.0.6:38291 (3 connections now open) m31003| Fri Jan 18 13:29:04.398 [initandlisten] connection accepted from 127.0.0.6:50125 #88 (4 connections now open) m31004| Fri Jan 18 13:29:04.417 [conn104] end connection 127.0.0.3:43602 (4 connections now open) m31004| Fri Jan 18 13:29:04.418 [initandlisten] connection accepted from 127.0.0.3:39153 #107 (5 connections now open) m31004| Fri Jan 18 13:29:05.090 [conn105] end connection 127.0.0.4:47604 (4 connections now open) m31004| Fri Jan 18 13:29:05.090 [initandlisten] connection accepted from 127.0.0.4:35529 #108 (5 connections now open) m31003| Fri Jan 18 13:29:05.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 60 more seconds m31003| Fri Jan 18 13:29:05.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 59 more seconds m31003| Fri Jan 18 13:29:05.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 59 more seconds m31003| Fri Jan 18 13:29:05.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 60 more seconds m31003| Fri Jan 18 13:29:05.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 59 more seconds m31003| Fri Jan 18 13:29:05.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 59 more seconds m31002| Fri Jan 18 13:29:05.444 [conn301] end connection 127.0.0.2:58567 (5 connections now open) m31002| Fri Jan 18 13:29:05.444 [initandlisten] connection accepted from 127.0.0.2:51896 #305 (6 connections now open) m31003| Fri Jan 18 13:29:05.940 [rsSync] replSet still syncing, not yet to minValid optime 50f991f5:10f5 m31001| Fri Jan 18 13:29:06.398 [conn394] end connection 127.0.0.6:45310 (4 connections now open) m31001| Fri Jan 18 13:29:06.398 [initandlisten] connection accepted from 127.0.0.6:50076 #398 (5 connections now open) m31002| Fri Jan 18 13:29:06.418 [conn302] end connection 127.0.0.3:35803 (5 connections now open) m31002| Fri Jan 18 13:29:06.418 [initandlisten] connection accepted from 127.0.0.3:53551 #306 (7 connections now open) m31003| Fri Jan 18 13:29:06.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 819, "optime" : { "t" : 1358533746000, "i" : 8731 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 785, "optime" : { "t" : 1358533704000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 785, "optime" : { "t" : 1358533704000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 785, "optime" : { "t" : 1358533745000, "i" : 2566 }, "optimeDate" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 784, "optime" : { "t" : 1358533746000, "i" : 3671 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 819, "optime" : { "t" : 1358533704000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 784, "optime" : { "t" : 1358533704000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 784, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:06Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 784, "optime" : { "t" : 1358533746000, "i" : 3660 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 784, "optime" : { "t" : 1358533745000, "i" : 241 }, "optimeDate" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 784, "optime" : { "t" : 1358533704000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 818, "optime" : { "t" : 1358533704000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 784, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:06Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 784, "optime" : { "t" : 1358533745000, "i" : 135 }, "optimeDate" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:06Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 785, "optime" : { "t" : 1358533704000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 4 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 785, "optime" : { "t" : 1358533704000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 20 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 817, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 785, "optime" : { "t" : 1358533745000, "i" : 6770 }, "optimeDate" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 784, "optime" : { "t" : 1358533746000, "i" : 3539 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 784, "optime" : { "t" : 1358533704000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:06Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 784, "optime" : { "t" : 1358533704000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 784, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 817, "optime" : { "t" : 1358533746000, "i" : 8731 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31001| Fri Jan 18 13:29:07.303 [conn395] end connection 127.0.0.4:37529 (4 connections now open) m31001| Fri Jan 18 13:29:07.303 [initandlisten] connection accepted from 127.0.0.4:52910 #399 (5 connections now open) m31003| Fri Jan 18 13:29:10.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 55 more seconds m31003| Fri Jan 18 13:29:10.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 54 more seconds m31003| Fri Jan 18 13:29:10.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 54 more seconds m31003| Fri Jan 18 13:29:10.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 55 more seconds m31003| Fri Jan 18 13:29:10.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 54 more seconds m31003| Fri Jan 18 13:29:10.192 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 54 more seconds m31004| Fri Jan 18 13:29:10.694 [conn102] end connection 127.0.0.5:33663 (4 connections now open) m31004| Fri Jan 18 13:29:11.194 [initandlisten] connection accepted from 127.0.0.5:42366 #109 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 825, "optime" : { "t" : 1358533752000, "i" : 1993 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 791, "optime" : { "t" : 1358533709000, "i" : 6179 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 791, "optime" : { "t" : 1358533709000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 791, "optime" : { "t" : 1358533751000, "i" : 3982 }, "optimeDate" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 790, "optime" : { "t" : 1358533750000, "i" : 3624 }, "optimeDate" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 825, "optime" : { "t" : 1358533709000, "i" : 6179 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 790, "optime" : { "t" : 1358533709000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:11Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 790, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:10Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 790, "optime" : { "t" : 1358533750000, "i" : 3624 }, "optimeDate" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 790, "optime" : { "t" : 1358533751000, "i" : 1107 }, "optimeDate" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 790, "optime" : { "t" : 1358533709000, "i" : 6179 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 824, "optime" : { "t" : 1358533709000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 790, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 790, "optime" : { "t" : 1358533751000, "i" : 1092 }, "optimeDate" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:12Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:28:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 791, "optime" : { "t" : 1358533709000, "i" : 6179 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 15 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 791, "optime" : { "t" : 1358533709000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:11Z"), "pingMs" : 12 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 823, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 791, "optime" : { "t" : 1358533751000, "i" : 1495 }, "optimeDate" : ISODate("2013-01-18T18:29:11Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 790, "optime" : { "t" : 1358533750000, "i" : 3624 }, "optimeDate" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 790, "optime" : { "t" : 1358533709000, "i" : 6179 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 790, "optime" : { "t" : 1358533709000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:11Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 790, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:11Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 823, "optime" : { "t" : 1358533752000, "i" : 1717 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:29:14.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:29:15.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 50 more seconds m31003| Fri Jan 18 13:29:15.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 49 more seconds m31003| Fri Jan 18 13:29:15.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 49 more seconds m31003| Fri Jan 18 13:29:15.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 50 more seconds m31003| Fri Jan 18 13:29:15.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 49 more seconds m31003| Fri Jan 18 13:29:15.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 49 more seconds m31001| Fri Jan 18 13:29:17.017 [conn396] end connection 127.0.0.5:44740 (4 connections now open) m31001| Fri Jan 18 13:29:17.093 [initandlisten] connection accepted from 127.0.0.5:55512 #400 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 830, "optime" : { "t" : 1358533757000, "i" : 4227 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 796, "optime" : { "t" : 1358533714000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:34Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:16Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 796, "optime" : { "t" : 1358533715000, "i" : 143 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:17Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 796, "optime" : { "t" : 1358533755000, "i" : 2393 }, "optimeDate" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:16Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 795, "optime" : { "t" : 1358533756000, "i" : 3213 }, "optimeDate" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 830, "optime" : { "t" : 1358533715000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 795, "optime" : { "t" : 1358533715000, "i" : 2001 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:17Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 795, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:17Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 795, "optime" : { "t" : 1358533756000, "i" : 3205 }, "optimeDate" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 795, "optime" : { "t" : 1358533756000, "i" : 8732 }, "optimeDate" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 795, "optime" : { "t" : 1358533715000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 829, "optime" : { "t" : 1358533715000, "i" : 2001 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 795, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 795, "optime" : { "t" : 1358533756000, "i" : 8732 }, "optimeDate" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:17Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 796, "optime" : { "t" : 1358533715000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 34 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 796, "optime" : { "t" : 1358533715000, "i" : 2001 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:17Z"), "pingMs" : 34 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 828, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 796, "optime" : { "t" : 1358533756000, "i" : 1167 }, "optimeDate" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 795, "optime" : { "t" : 1358533756000, "i" : 2728 }, "optimeDate" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 795, "optime" : { "t" : 1358533715000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:16Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 795, "optime" : { "t" : 1358533715000, "i" : 2001 }, "optimeDate" : ISODate("2013-01-18T18:28:35Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:17Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 795, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 828, "optime" : { "t" : 1358533757000, "i" : 4227 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:29:20.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 45 more seconds m31003| Fri Jan 18 13:29:20.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 44 more seconds m31003| Fri Jan 18 13:29:20.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 44 more seconds m31003| Fri Jan 18 13:29:20.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 45 more seconds m31003| Fri Jan 18 13:29:20.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 44 more seconds m31003| Fri Jan 18 13:29:20.193 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 44 more seconds m31002| Fri Jan 18 13:29:20.399 [conn303] end connection 127.0.0.6:44375 (5 connections now open) m31002| Fri Jan 18 13:29:20.400 [initandlisten] connection accepted from 127.0.0.6:43126 #307 (6 connections now open) m31003| Fri Jan 18 13:29:20.420 [conn86] end connection 127.0.0.3:60091 (3 connections now open) m31003| Fri Jan 18 13:29:20.420 [initandlisten] connection accepted from 127.0.0.3:43985 #89 (4 connections now open) m31003| Fri Jan 18 13:29:21.092 [conn87] end connection 127.0.0.4:39688 (3 connections now open) m31003| Fri Jan 18 13:29:21.092 [initandlisten] connection accepted from 127.0.0.4:47737 #90 (4 connections now open) m31001| Fri Jan 18 13:29:21.447 [conn397] end connection 127.0.0.2:44416 (4 connections now open) m31001| Fri Jan 18 13:29:21.447 [initandlisten] connection accepted from 127.0.0.2:35737 #401 (6 connections now open) m31000| Fri Jan 18 13:29:22.400 [conn108] end connection 127.0.0.6:59669 (11 connections now open) m31000| Fri Jan 18 13:29:22.400 [initandlisten] connection accepted from 127.0.0.6:52532 #111 (12 connections now open) m31000| Fri Jan 18 13:29:22.420 [conn109] end connection 127.0.0.3:44016 (11 connections now open) m31000| Fri Jan 18 13:29:22.420 [initandlisten] connection accepted from 127.0.0.3:34895 #112 (12 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 835, "optime" : { "t" : 1358533762000, "i" : 5895 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 801, "optime" : { "t" : 1358533720000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 801, "optime" : { "t" : 1358533720000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 801, "optime" : { "t" : 1358533761000, "i" : 3744 }, "optimeDate" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 800, "optime" : { "t" : 1358533762000, "i" : 3535 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 835, "optime" : { "t" : 1358533720000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 800, "optime" : { "t" : 1358533720000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 800, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 800, "optime" : { "t" : 1358533762000, "i" : 3515 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 800, "optime" : { "t" : 1358533761000, "i" : 1102 }, "optimeDate" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:21Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 800, "optime" : { "t" : 1358533720000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 834, "optime" : { "t" : 1358533720000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 800, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 800, "optime" : { "t" : 1358533761000, "i" : 1087 }, "optimeDate" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:22Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 801, "optime" : { "t" : 1358533720000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:22Z"), "pingMs" : 30 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 801, "optime" : { "t" : 1358533720000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:21Z"), "pingMs" : 24 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 833, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 801, "optime" : { "t" : 1358533761000, "i" : 1843 }, "optimeDate" : ISODate("2013-01-18T18:29:21Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 800, "optime" : { "t" : 1358533762000, "i" : 3180 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 800, "optime" : { "t" : 1358533720000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:22Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 800, "optime" : { "t" : 1358533720000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:40Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 800, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 833, "optime" : { "t" : 1358533762000, "i" : 5895 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:29:23.092 [conn110] end connection 127.0.0.4:39487 (11 connections now open) m31000| Fri Jan 18 13:29:23.093 [initandlisten] connection accepted from 127.0.0.4:36326 #113 (12 connections now open) m31000| Fri Jan 18 13:29:24.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31003| Fri Jan 18 13:29:25.194 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 40 more seconds m31003| Fri Jan 18 13:29:25.194 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 39 more seconds m31003| Fri Jan 18 13:29:25.194 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 39 more seconds m31003| Fri Jan 18 13:29:25.194 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 40 more seconds m31003| Fri Jan 18 13:29:25.194 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 39 more seconds m31003| Fri Jan 18 13:29:25.194 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 39 more seconds m31000| Fri Jan 18 13:29:27.350 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:230650 115ms STATUS FROM : 127.0.0.2:31000 m31000| Fri Jan 18 13:29:27.928 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:232756 116ms { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 840, "optime" : { "t" : 1358533767000, "i" : 8176 }, "optimeDate" : ISODate("2013-01-18T18:29:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 806, "optime" : { "t" : 1358533725000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 806, "optime" : { "t" : 1358533725000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 806, "optime" : { "t" : 1358533767000, "i" : 4280 }, "optimeDate" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 805, "optime" : { "t" : 1358533766000, "i" : 3612 }, "optimeDate" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 840, "optime" : { "t" : 1358533725000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 805, "optime" : { "t" : 1358533725000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:27Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 805, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:27Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 805, "optime" : { "t" : 1358533766000, "i" : 3612 }, "optimeDate" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 805, "optime" : { "t" : 1358533767000, "i" : 891 }, "optimeDate" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 805, "optime" : { "t" : 1358533725000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 839, "optime" : { "t" : 1358533725000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 805, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 805, "optime" : { "t" : 1358533767000, "i" : 891 }, "optimeDate" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:27Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 806, "optime" : { "t" : 1358533725000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 26 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 806, "optime" : { "t" : 1358533725000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:27Z"), "pingMs" : 31 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 838, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 806, "optime" : { "t" : 1358533766000, "i" : 1257 }, "optimeDate" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 805, "optime" : { "t" : 1358533766000, "i" : 3612 }, "optimeDate" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 805, "optime" : { "t" : 1358533725000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:26Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 805, "optime" : { "t" : 1358533725000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:45Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:27Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 805, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 838, "optime" : { "t" : 1358533767000, "i" : 8176 }, "optimeDate" : ISODate("2013-01-18T18:29:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:29:28.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31003| Fri Jan 18 13:29:30.194 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 35 more seconds m31003| Fri Jan 18 13:29:30.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 34 more seconds m31003| Fri Jan 18 13:29:30.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 34 more seconds m31003| Fri Jan 18 13:29:30.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 35 more seconds m31003| Fri Jan 18 13:29:30.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 34 more seconds m31003| Fri Jan 18 13:29:30.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 34 more seconds m31002| Fri Jan 18 13:29:30.830 [conn304] end connection 127.0.0.5:49286 (5 connections now open) m31002| Fri Jan 18 13:29:30.856 [initandlisten] connection accepted from 127.0.0.5:50605 #308 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 846, "optime" : { "t" : 1358533773000, "i" : 2470 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 812, "optime" : { "t" : 1358533730000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 812, "optime" : { "t" : 1358533730000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:33Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 812, "optime" : { "t" : 1358533771000, "i" : 3453 }, "optimeDate" : ISODate("2013-01-18T18:29:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:32Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 811, "optime" : { "t" : 1358533772000, "i" : 3821 }, "optimeDate" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 846, "optime" : { "t" : 1358533731000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:51Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 811, "optime" : { "t" : 1358533731000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 811, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:33Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 811, "optime" : { "t" : 1358533772000, "i" : 3761 }, "optimeDate" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 811, "optime" : { "t" : 1358533773000, "i" : 1255 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 811, "optime" : { "t" : 1358533730000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:28:50Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 845, "optime" : { "t" : 1358533731000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:51Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 811, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 811, "optime" : { "t" : 1358533773000, "i" : 1237 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:33Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 812, "optime" : { "t" : 1358533731000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 17 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 812, "optime" : { "t" : 1358533731000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:33Z"), "pingMs" : 27 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 844, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 812, "optime" : { "t" : 1358533771000, "i" : 1363 }, "optimeDate" : ISODate("2013-01-18T18:29:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 811, "optime" : { "t" : 1358533772000, "i" : 3233 }, "optimeDate" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 811, "optime" : { "t" : 1358533731000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:32Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 811, "optime" : { "t" : 1358533731000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:51Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:33Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 811, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 844, "optime" : { "t" : 1358533773000, "i" : 2470 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:29:33.448 [conn106] end connection 127.0.0.2:43392 (4 connections now open) m31004| Fri Jan 18 13:29:33.448 [initandlisten] connection accepted from 127.0.0.2:53036 #110 (5 connections now open) m31003| Fri Jan 18 13:29:34.401 [conn88] end connection 127.0.0.6:50125 (3 connections now open) m31003| Fri Jan 18 13:29:34.402 [initandlisten] connection accepted from 127.0.0.6:32988 #91 (4 connections now open) m31004| Fri Jan 18 13:29:34.421 [conn107] end connection 127.0.0.3:39153 (4 connections now open) m31004| Fri Jan 18 13:29:34.422 [initandlisten] connection accepted from 127.0.0.3:51981 #111 (5 connections now open) m31004| Fri Jan 18 13:29:35.094 [conn108] end connection 127.0.0.4:35529 (4 connections now open) m31004| Fri Jan 18 13:29:35.094 [initandlisten] connection accepted from 127.0.0.4:48747 #112 (5 connections now open) m31003| Fri Jan 18 13:29:35.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 30 more seconds m31003| Fri Jan 18 13:29:35.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 29 more seconds m31003| Fri Jan 18 13:29:35.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 29 more seconds m31003| Fri Jan 18 13:29:35.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 30 more seconds m31003| Fri Jan 18 13:29:35.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 29 more seconds m31003| Fri Jan 18 13:29:35.195 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 29 more seconds m31002| Fri Jan 18 13:29:35.449 [conn305] end connection 127.0.0.2:51896 (5 connections now open) m31002| Fri Jan 18 13:29:35.449 [initandlisten] connection accepted from 127.0.0.2:55073 #309 (6 connections now open) m31001| Fri Jan 18 13:29:36.402 [conn398] end connection 127.0.0.6:50076 (4 connections now open) m31001| Fri Jan 18 13:29:36.402 [initandlisten] connection accepted from 127.0.0.6:45408 #402 (5 connections now open) m31002| Fri Jan 18 13:29:36.422 [conn306] end connection 127.0.0.3:53551 (5 connections now open) m31002| Fri Jan 18 13:29:36.422 [initandlisten] connection accepted from 127.0.0.3:60691 #310 (6 connections now open) m31001| Fri Jan 18 13:29:37.307 [conn399] end connection 127.0.0.4:52910 (4 connections now open) m31001| Fri Jan 18 13:29:37.307 [initandlisten] connection accepted from 127.0.0.4:35881 #403 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31003| Fri Jan 18 13:29:38.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 851, "optime" : { "t" : 1358533778000, "i" : 5250 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 817, "optime" : { "t" : 1358533736000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:38Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 817, "optime" : { "t" : 1358533736000, "i" : 1287 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 817, "optime" : { "t" : 1358533777000, "i" : 3695 }, "optimeDate" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:38Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 816, "optime" : { "t" : 1358533778000, "i" : 3842 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 851, "optime" : { "t" : 1358533736000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 816, "optime" : { "t" : 1358533736000, "i" : 1287 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 816, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 816, "optime" : { "t" : 1358533778000, "i" : 3836 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 816, "optime" : { "t" : 1358533777000, "i" : 408 }, "optimeDate" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:37Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 816, "optime" : { "t" : 1358533736000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:38Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 850, "optime" : { "t" : 1358533736000, "i" : 1287 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 816, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 816, "optime" : { "t" : 1358533777000, "i" : 403 }, "optimeDate" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:38Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:38Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 817, "optime" : { "t" : 1358533736000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:38Z"), "pingMs" : 25 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 817, "optime" : { "t" : 1358533736000, "i" : 1287 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:37Z"), "pingMs" : 31 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 849, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 817, "optime" : { "t" : 1358533773000, "i" : 2470 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 816, "optime" : { "t" : 1358533778000, "i" : 3723 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 816, "optime" : { "t" : 1358533736000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:38Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 816, "optime" : { "t" : 1358533736000, "i" : 1287 }, "optimeDate" : ISODate("2013-01-18T18:28:56Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 816, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 849, "optime" : { "t" : 1358533778000, "i" : 5250 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31000| Fri Jan 18 13:29:39.601 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:148594 nreturned:160000 reslen:2720020 148ms m31000| Fri Jan 18 13:29:39.705 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:137225 nreturned:160000 reslen:2720020 137ms m31003| Fri Jan 18 13:29:40.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 25 more seconds m31003| Fri Jan 18 13:29:40.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 24 more seconds m31003| Fri Jan 18 13:29:40.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 24 more seconds m31003| Fri Jan 18 13:29:40.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 25 more seconds m31003| Fri Jan 18 13:29:40.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 24 more seconds m31003| Fri Jan 18 13:29:40.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 24 more seconds STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 856, "optime" : { "t" : 1358533783000, "i" : 9575 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 822, "optime" : { "t" : 1358533741000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 822, "optime" : { "t" : 1358533741000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 822, "optime" : { "t" : 1358533783000, "i" : 4604 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 821, "optime" : { "t" : 1358533782000, "i" : 3090 }, "optimeDate" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 856, "optime" : { "t" : 1358533741000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 821, "optime" : { "t" : 1358533741000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:43Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 821, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:43Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 821, "optime" : { "t" : 1358533782000, "i" : 3090 }, "optimeDate" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 821, "optime" : { "t" : 1358533783000, "i" : 2 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 821, "optime" : { "t" : 1358533741000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 855, "optime" : { "t" : 1358533741000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 821, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 821, "optime" : { "t" : 1358533782000, "i" : 7475 }, "optimeDate" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:43Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 822, "optime" : { "t" : 1358533741000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 24 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 822, "optime" : { "t" : 1358533741000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:43Z"), "pingMs" : 24 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 854, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 822, "optime" : { "t" : 1358533783000, "i" : 7870 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 821, "optime" : { "t" : 1358533782000, "i" : 3090 }, "optimeDate" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 821, "optime" : { "t" : 1358533741000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:42Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 821, "optime" : { "t" : 1358533741000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:01Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:43Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 821, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:43Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 854, "optime" : { "t" : 1358533783000, "i" : 9201 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:29:45.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 20 more seconds m31003| Fri Jan 18 13:29:45.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 19 more seconds m31003| Fri Jan 18 13:29:45.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 19 more seconds m31003| Fri Jan 18 13:29:45.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 20 more seconds m31003| Fri Jan 18 13:29:45.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 19 more seconds m31003| Fri Jan 18 13:29:45.196 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 19 more seconds m31003| Fri Jan 18 13:29:46.199 [rsMgr] replSet I don't see a primary and I can't elect myself m31000| Fri Jan 18 13:29:46.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31001| Fri Jan 18 13:29:47.482 [conn400] end connection 127.0.0.5:55512 (4 connections now open) m31001| Fri Jan 18 13:29:47.482 [initandlisten] connection accepted from 127.0.0.5:52789 #404 (6 connections now open) m31004| Fri Jan 18 13:29:48.699 [conn109] end connection 127.0.0.5:42366 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 862, "optime" : { "t" : 1358533789000, "i" : 2798 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 828, "optime" : { "t" : 1358533746000, "i" : 5558 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 828, "optime" : { "t" : 1358533746000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:49Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 828, "optime" : { "t" : 1358533787000, "i" : 3689 }, "optimeDate" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:48Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 827, "optime" : { "t" : 1358533788000, "i" : 3939 }, "optimeDate" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 862, "optime" : { "t" : 1358533746000, "i" : 5558 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 827, "optime" : { "t" : 1358533746000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 827, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 827, "optime" : { "t" : 1358533788000, "i" : 3596 }, "optimeDate" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 827, "optime" : { "t" : 1358533789000, "i" : 1283 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 827, "optime" : { "t" : 1358533746000, "i" : 5558 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 861, "optime" : { "t" : 1358533746000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 827, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:49Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 827, "optime" : { "t" : 1358533789000, "i" : 1235 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:49Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 828, "optime" : { "t" : 1358533746000, "i" : 5558 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:48Z"), "pingMs" : 32 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 828, "optime" : { "t" : 1358533746000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 17 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 860, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 828, "optime" : { "t" : 1358533783000, "i" : 9575 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 827, "optime" : { "t" : 1358533788000, "i" : 3771 }, "optimeDate" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 827, "optime" : { "t" : 1358533746000, "i" : 5558 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:48Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 827, "optime" : { "t" : 1358533746000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:06Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:49Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 827, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 860, "optime" : { "t" : 1358533789000, "i" : 1309 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:29:49.199 [initandlisten] connection accepted from 127.0.0.5:45425 #113 (5 connections now open) m31003| Fri Jan 18 13:29:50.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 15 more seconds m31003| Fri Jan 18 13:29:50.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 14 more seconds m31003| Fri Jan 18 13:29:50.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 14 more seconds m31003| Fri Jan 18 13:29:50.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 15 more seconds m31003| Fri Jan 18 13:29:50.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 14 more seconds m31003| Fri Jan 18 13:29:50.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 14 more seconds m31002| Fri Jan 18 13:29:50.403 [conn307] end connection 127.0.0.6:43126 (5 connections now open) m31002| Fri Jan 18 13:29:50.404 [initandlisten] connection accepted from 127.0.0.6:52544 #311 (6 connections now open) m31003| Fri Jan 18 13:29:50.424 [conn89] end connection 127.0.0.3:43985 (3 connections now open) m31003| Fri Jan 18 13:29:50.424 [initandlisten] connection accepted from 127.0.0.3:33044 #92 (5 connections now open) m31003| Fri Jan 18 13:29:51.096 [conn90] end connection 127.0.0.4:47737 (3 connections now open) m31003| Fri Jan 18 13:29:51.097 [initandlisten] connection accepted from 127.0.0.4:44572 #93 (4 connections now open) m31001| Fri Jan 18 13:29:51.451 [conn401] end connection 127.0.0.2:35737 (4 connections now open) m31001| Fri Jan 18 13:29:51.451 [initandlisten] connection accepted from 127.0.0.2:42463 #405 (5 connections now open) m31000| Fri Jan 18 13:29:52.404 [conn111] end connection 127.0.0.6:52532 (11 connections now open) m31000| Fri Jan 18 13:29:52.404 [initandlisten] connection accepted from 127.0.0.6:49598 #114 (12 connections now open) m31000| Fri Jan 18 13:29:52.424 [conn112] end connection 127.0.0.3:34895 (11 connections now open) m31000| Fri Jan 18 13:29:52.424 [initandlisten] connection accepted from 127.0.0.3:59287 #115 (12 connections now open) m31000| Fri Jan 18 13:29:53.097 [conn113] end connection 127.0.0.4:36326 (11 connections now open) m31000| Fri Jan 18 13:29:53.097 [initandlisten] connection accepted from 127.0.0.4:48090 #116 (12 connections now open) m31000| Fri Jan 18 13:29:53.938 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:1959512 979ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 867, "optime" : { "t" : 1358533794000, "i" : 11226 }, "optimeDate" : ISODate("2013-01-18T18:29:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 833, "optime" : { "t" : 1358533752000, "i" : 1075 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 833, "optime" : { "t" : 1358533752000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 833, "optime" : { "t" : 1358533792000, "i" : 7157 }, "optimeDate" : ISODate("2013-01-18T18:29:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:54Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 832, "optime" : { "t" : 1358533794000, "i" : 6429 }, "optimeDate" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 867, "optime" : { "t" : 1358533752000, "i" : 1075 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 832, "optime" : { "t" : 1358533752000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 832, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 832, "optime" : { "t" : 1358533794000, "i" : 6412 }, "optimeDate" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 832, "optime" : { "t" : 1358533792000, "i" : 7158 }, "optimeDate" : ISODate("2013-01-18T18:29:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:53Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 832, "optime" : { "t" : 1358533752000, "i" : 1075 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 866, "optime" : { "t" : 1358533752000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 832, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:53Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 832, "optime" : { "t" : 1358533792000, "i" : 7157 }, "optimeDate" : ISODate("2013-01-18T18:29:52Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:54Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:54Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 833, "optime" : { "t" : 1358533752000, "i" : 1075 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:54Z"), "pingMs" : 35 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 833, "optime" : { "t" : 1358533752000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 22 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 865, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 833, "optime" : { "t" : 1358533794000, "i" : 3645 }, "optimeDate" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:29:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 832, "optime" : { "t" : 1358533794000, "i" : 6404 }, "optimeDate" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 832, "optime" : { "t" : 1358533752000, "i" : 1075 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:54Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 832, "optime" : { "t" : 1358533752000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:12Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 832, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 865, "optime" : { "t" : 1358533794000, "i" : 11226 }, "optimeDate" : ISODate("2013-01-18T18:29:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:29:55.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 10 more seconds m31003| Fri Jan 18 13:29:55.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 9 more seconds m31003| Fri Jan 18 13:29:55.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 9 more seconds m31003| Fri Jan 18 13:29:55.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 10 more seconds m31003| Fri Jan 18 13:29:55.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 9 more seconds m31003| Fri Jan 18 13:29:55.197 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 9 more seconds m31000| Fri Jan 18 13:29:56.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31000| Fri Jan 18 13:29:58.234 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:360833 180ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 873, "optime" : { "t" : 1358533800000, "i" : 581 }, "optimeDate" : ISODate("2013-01-18T18:30:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 839, "optime" : { "t" : 1358533757000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 839, "optime" : { "t" : 1358533757000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 839, "optime" : { "t" : 1358533799000, "i" : 3819 }, "optimeDate" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 838, "optime" : { "t" : 1358533798000, "i" : 2899 }, "optimeDate" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 873, "optime" : { "t" : 1358533757000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 838, "optime" : { "t" : 1358533757000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:59Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 838, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:59Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 838, "optime" : { "t" : 1358533798000, "i" : 2869 }, "optimeDate" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 838, "optime" : { "t" : 1358533799000, "i" : 737 }, "optimeDate" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 838, "optime" : { "t" : 1358533757000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 872, "optime" : { "t" : 1358533757000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 838, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:59Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 838, "optime" : { "t" : 1358533799000, "i" : 715 }, "optimeDate" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:00Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 839, "optime" : { "t" : 1358533757000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 17 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 839, "optime" : { "t" : 1358533757000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 33 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 871, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f991f5:10f5", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 839, "optime" : { "t" : 1358533799000, "i" : 1248 }, "optimeDate" : ISODate("2013-01-18T18:29:59Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 838, "optime" : { "t" : 1358533798000, "i" : 2503 }, "optimeDate" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 838, "optime" : { "t" : 1358533757000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:58Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 838, "optime" : { "t" : 1358533757000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:17Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:59Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 838, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:29:59Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 871, "optime" : { "t" : 1358533800000, "i" : 581 }, "optimeDate" : ISODate("2013-01-18T18:30:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:30:00.198 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 5 more seconds m31003| Fri Jan 18 13:30:00.198 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 4 more seconds m31003| Fri Jan 18 13:30:00.198 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 4 more seconds m31003| Fri Jan 18 13:30:00.198 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 5 more seconds m31003| Fri Jan 18 13:30:00.198 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 4 more seconds m31003| Fri Jan 18 13:30:00.198 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 4 more seconds m31003| Fri Jan 18 13:30:00.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Fri Jan 18 13:30:01.331 [conn308] end connection 127.0.0.5:50605 (5 connections now open) m31002| Fri Jan 18 13:30:01.331 [initandlisten] connection accepted from 127.0.0.5:51398 #312 (6 connections now open) m31000| Fri Jan 18 13:30:03.298 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:306293 153ms m31004| Fri Jan 18 13:30:03.453 [conn110] end connection 127.0.0.2:53036 (4 connections now open) m31004| Fri Jan 18 13:30:03.453 [initandlisten] connection accepted from 127.0.0.2:48835 #114 (5 connections now open) m31003| Fri Jan 18 13:30:04.406 [conn91] end connection 127.0.0.6:32988 (3 connections now open) m31003| Fri Jan 18 13:30:04.406 [initandlisten] connection accepted from 127.0.0.6:47365 #94 (4 connections now open) m31004| Fri Jan 18 13:30:04.425 [conn111] end connection 127.0.0.3:51981 (4 connections now open) m31004| Fri Jan 18 13:30:04.425 [initandlisten] connection accepted from 127.0.0.3:54162 #115 (5 connections now open) m31000| Fri Jan 18 13:30:04.626 [journal] old journal file will be removed: /data/db/testReplSet-0/journal/j._0 m31000| Fri Jan 18 13:30:04.702 [conn22] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:312214 156ms m31004| Fri Jan 18 13:30:05.098 [conn112] end connection 127.0.0.4:48747 (4 connections now open) m31004| Fri Jan 18 13:30:05.098 [initandlisten] connection accepted from 127.0.0.4:59048 #116 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31003| Fri Jan 18 13:30:05.198 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 0 more seconds m31003| Fri Jan 18 13:30:05.198 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:30:05.198 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.3:31001 m31003| Fri Jan 18 13:30:05.198 [rsBackgroundSync] SYNCED TO 127.0.0.3:31001 m31003| Fri Jan 18 13:30:05.198 [rsBackgroundSync] replSet syncing to: 127.0.0.3:31001 m31001| Fri Jan 18 13:30:05.219 [initandlisten] connection accepted from 127.0.0.5:54196 #406 (6 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 878, "optime" : { "t" : 1358533805000, "i" : 1726 }, "optimeDate" : ISODate("2013-01-18T18:30:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 844, "optime" : { "t" : 1358533762000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:04Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 844, "optime" : { "t" : 1358533762000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:05Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 844, "optime" : { "t" : 1358533803000, "i" : 1418 }, "optimeDate" : ISODate("2013-01-18T18:30:03Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:04Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 843, "optime" : { "t" : 1358533804000, "i" : 3125 }, "optimeDate" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 878, "optime" : { "t" : 1358533762000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 843, "optime" : { "t" : 1358533762000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 843, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 843, "optime" : { "t" : 1358533804000, "i" : 2984 }, "optimeDate" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 843, "optime" : { "t" : 1358533805000, "i" : 1087 }, "optimeDate" : ISODate("2013-01-18T18:30:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 843, "optime" : { "t" : 1358533762000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:04Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 877, "optime" : { "t" : 1358533762000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 843, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 843, "optime" : { "t" : 1358533805000, "i" : 1071 }, "optimeDate" : ISODate("2013-01-18T18:30:05Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:05Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 844, "optime" : { "t" : 1358533762000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:04Z"), "pingMs" : 23 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 844, "optime" : { "t" : 1358533762000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:05Z"), "pingMs" : 39 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 876, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "syncing to: 127.0.0.3:31001", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 844, "optime" : { "t" : 1358533804000, "i" : 1650 }, "optimeDate" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 843, "optime" : { "t" : 1358533804000, "i" : 3125 }, "optimeDate" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 843, "optime" : { "t" : 1358533762000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:04Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 843, "optime" : { "t" : 1358533762000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:22Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:05Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 843, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f991f5:10f5" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 876, "optime" : { "t" : 1358533805000, "i" : 1406 }, "optimeDate" : ISODate("2013-01-18T18:30:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:30:05.270 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 0 more seconds m31003| Fri Jan 18 13:30:05.270 [rsBackgroundSync] CLOSEST CANDIDATE IS : 127.0.0.4:31002 m31003| Fri Jan 18 13:30:05.270 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 600 more seconds m31003| Fri Jan 18 13:30:05.270 [rsBackgroundSync] SYNCED TO 127.0.0.4:31002 m31003| Fri Jan 18 13:30:05.270 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31001| Fri Jan 18 13:30:05.270 [conn406] end connection 127.0.0.5:54196 (5 connections now open) m31002| Fri Jan 18 13:30:05.306 [initandlisten] connection accepted from 127.0.0.5:57165 #313 (7 connections now open) m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 0 more seconds m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 600 more seconds m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 600 more seconds m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet not trying to sync from 127.0.0.6:31004, it is vetoed for 0 more seconds m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet not trying to sync from 127.0.0.4:31002, it is vetoed for 600 more seconds m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet not trying to sync from 127.0.0.3:31001, it is vetoed for 600 more seconds m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet error RS102 too stale to catch up, at least from 127.0.0.4:31002 m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet our last optime : Jan 18 13:16:55 50f99197:14b4 m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet oldest at 127.0.0.4:31002 : Jan 18 13:27:48 50f99424:19cf m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember m31003| Fri Jan 18 13:30:05.306 [rsBackgroundSync] replSet error RS102 too stale to catch up m31002| Fri Jan 18 13:30:05.339 [conn313] end connection 127.0.0.5:57165 (6 connections now open) m31002| Fri Jan 18 13:30:05.453 [conn309] end connection 127.0.0.2:55073 (5 connections now open) m31002| Fri Jan 18 13:30:05.453 [initandlisten] connection accepted from 127.0.0.2:46240 #314 (6 connections now open) m31003| Fri Jan 18 13:30:05.948 [rsSync] replSet still syncing, not yet to minValid optime 50f99424:19cf m31001| Fri Jan 18 13:30:06.406 [conn402] end connection 127.0.0.6:45408 (4 connections now open) m31001| Fri Jan 18 13:30:06.406 [initandlisten] connection accepted from 127.0.0.6:49943 #407 (5 connections now open) m31002| Fri Jan 18 13:30:06.425 [conn310] end connection 127.0.0.3:60691 (5 connections now open) m31002| Fri Jan 18 13:30:06.425 [initandlisten] connection accepted from 127.0.0.3:39648 #315 (6 connections now open) m31001| Fri Jan 18 13:30:07.312 [conn403] end connection 127.0.0.4:35881 (4 connections now open) m31001| Fri Jan 18 13:30:07.312 [initandlisten] connection accepted from 127.0.0.4:54728 #408 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 883, "optime" : { "t" : 1358533810000, "i" : 3793 }, "optimeDate" : ISODate("2013-01-18T18:30:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 849, "optime" : { "t" : 1358533767000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 849, "optime" : { "t" : 1358533767000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 849, "optime" : { "t" : 1358533809000, "i" : 2526 }, "optimeDate" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:10Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 848, "optime" : { "t" : 1358533810000, "i" : 3637 }, "optimeDate" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 883, "optime" : { "t" : 1358533770000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:30Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 848, "optime" : { "t" : 1358533770000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 848, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 848, "optime" : { "t" : 1358533810000, "i" : 3583 }, "optimeDate" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 848, "optime" : { "t" : 1358533809000, "i" : 353 }, "optimeDate" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 848, "optime" : { "t" : 1358533767000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 882, "optime" : { "t" : 1358533770000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:30Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 848, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:09Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 848, "optime" : { "t" : 1358533809000, "i" : 348 }, "optimeDate" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:10Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:10Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 849, "optime" : { "t" : 1358533767000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:10Z"), "pingMs" : 37 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 849, "optime" : { "t" : 1358533767000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:27Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 27 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 881, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f99424:19cf", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 849, "optime" : { "t" : 1358533809000, "i" : 1590 }, "optimeDate" : ISODate("2013-01-18T18:30:09Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 848, "optime" : { "t" : 1358533810000, "i" : 3205 }, "optimeDate" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 848, "optime" : { "t" : 1358533770000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:10Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 848, "optime" : { "t" : 1358533770000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 848, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 881, "optime" : { "t" : 1358533810000, "i" : 3793 }, "optimeDate" : ISODate("2013-01-18T18:30:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:30:10.599 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 888, "optime" : { "t" : 1358533815000, "i" : 6874 }, "optimeDate" : ISODate("2013-01-18T18:30:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 854, "optime" : { "t" : 1358533773000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 854, "optime" : { "t" : 1358533773000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 854, "optime" : { "t" : 1358533815000, "i" : 3635 }, "optimeDate" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 853, "optime" : { "t" : 1358533814000, "i" : 3486 }, "optimeDate" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 888, "optime" : { "t" : 1358533773000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 853, "optime" : { "t" : 1358533773000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:15Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 853, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 853, "optime" : { "t" : 1358533814000, "i" : 3008 }, "optimeDate" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 853, "optime" : { "t" : 1358533815000, "i" : 1171 }, "optimeDate" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 853, "optime" : { "t" : 1358533773000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 887, "optime" : { "t" : 1358533773000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 853, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:15Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 853, "optime" : { "t" : 1358533815000, "i" : 1156 }, "optimeDate" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:15Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:29:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 854, "optime" : { "t" : 1358533773000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:14Z"), "pingMs" : 23 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 854, "optime" : { "t" : 1358533773000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 16 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 886, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f99424:19cf", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 854, "optime" : { "t" : 1358533814000, "i" : 1725 }, "optimeDate" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 853, "optime" : { "t" : 1358533814000, "i" : 3075 }, "optimeDate" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 853, "optime" : { "t" : 1358533773000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:14Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 853, "optime" : { "t" : 1358533773000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:33Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:15Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 853, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 886, "optime" : { "t" : 1358533815000, "i" : 6733 }, "optimeDate" : ISODate("2013-01-18T18:30:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31001| Fri Jan 18 13:30:18.007 [conn404] end connection 127.0.0.5:52789 (4 connections now open) m31001| Fri Jan 18 13:30:18.007 [initandlisten] connection accepted from 127.0.0.5:38292 #409 (6 connections now open) m31000| Fri Jan 18 13:30:18.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 m31004| Fri Jan 18 13:30:19.375 [journal] old journal file will be removed: /data/db/testReplSet-4/journal/j._0 m31002| Fri Jan 18 13:30:20.407 [conn311] end connection 127.0.0.6:52544 (5 connections now open) m31002| Fri Jan 18 13:30:20.407 [initandlisten] connection accepted from 127.0.0.6:42837 #316 (6 connections now open) m31003| Fri Jan 18 13:30:20.427 [conn92] end connection 127.0.0.3:33044 (3 connections now open) m31003| Fri Jan 18 13:30:20.427 [initandlisten] connection accepted from 127.0.0.3:50241 #95 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 894, "optime" : { "t" : 1358533821000, "i" : 69 }, "optimeDate" : ISODate("2013-01-18T18:30:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 860, "optime" : { "t" : 1358533778000, "i" : 1843 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:20Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 860, "optime" : { "t" : 1358533778000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 860, "optime" : { "t" : 1358533819000, "i" : 2833 }, "optimeDate" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:20Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 859, "optime" : { "t" : 1358533820000, "i" : 3597 }, "optimeDate" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 894, "optime" : { "t" : 1358533778000, "i" : 1843 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 859, "optime" : { "t" : 1358533778000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 859, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 859, "optime" : { "t" : 1358533820000, "i" : 3547 }, "optimeDate" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 859, "optime" : { "t" : 1358533819000, "i" : 1337 }, "optimeDate" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:19Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 859, "optime" : { "t" : 1358533778000, "i" : 1843 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:20Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 893, "optime" : { "t" : 1358533778000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 859, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:19Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 859, "optime" : { "t" : 1358533819000, "i" : 1326 }, "optimeDate" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:20Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:21Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 860, "optime" : { "t" : 1358533778000, "i" : 1843 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:20Z"), "pingMs" : 46 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 860, "optime" : { "t" : 1358533778000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 29 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 892, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f99424:19cf", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 860, "optime" : { "t" : 1358533819000, "i" : 1773 }, "optimeDate" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 859, "optime" : { "t" : 1358533820000, "i" : 3460 }, "optimeDate" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 859, "optime" : { "t" : 1358533778000, "i" : 1843 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:20Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 859, "optime" : { "t" : 1358533778000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:38Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 859, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 892, "optime" : { "t" : 1358533821000, "i" : 63 }, "optimeDate" : ISODate("2013-01-18T18:30:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31003| Fri Jan 18 13:30:21.101 [conn93] end connection 127.0.0.4:44572 (3 connections now open) m31003| Fri Jan 18 13:30:21.101 [initandlisten] connection accepted from 127.0.0.4:35907 #96 (4 connections now open) m31001| Fri Jan 18 13:30:21.455 [conn405] end connection 127.0.0.2:42463 (4 connections now open) m31001| Fri Jan 18 13:30:21.455 [initandlisten] connection accepted from 127.0.0.2:38363 #410 (6 connections now open) m31000| Fri Jan 18 13:30:21.604 [conn32] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313270917661975 ntoreturn:0 keyUpdates:0 locks(micros) r:135200 nreturned:160000 reslen:2720020 135ms m31000| Fri Jan 18 13:30:21.719 [conn33] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358532961000|1 } } cursorid:313272356675815 ntoreturn:0 keyUpdates:0 locks(micros) r:131232 nreturned:160000 reslen:2720020 131ms m31000| Fri Jan 18 13:30:22.408 [conn114] end connection 127.0.0.6:49598 (11 connections now open) m31000| Fri Jan 18 13:30:22.408 [initandlisten] connection accepted from 127.0.0.6:33719 #117 (12 connections now open) m31000| Fri Jan 18 13:30:22.427 [conn115] end connection 127.0.0.3:59287 (11 connections now open) m31000| Fri Jan 18 13:30:22.428 [initandlisten] connection accepted from 127.0.0.3:46270 #118 (12 connections now open) m31000| Fri Jan 18 13:30:23.101 [conn116] end connection 127.0.0.4:48090 (11 connections now open) m31000| Fri Jan 18 13:30:23.101 [initandlisten] connection accepted from 127.0.0.4:44093 #119 (12 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 899, "optime" : { "t" : 1358533826000, "i" : 3652 }, "optimeDate" : ISODate("2013-01-18T18:30:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 865, "optime" : { "t" : 1358533783000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 865, "optime" : { "t" : 1358533783000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 865, "optime" : { "t" : 1358533825000, "i" : 3779 }, "optimeDate" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 864, "optime" : { "t" : 1358533824000, "i" : 3512 }, "optimeDate" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:25Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 899, "optime" : { "t" : 1358533783000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 864, "optime" : { "t" : 1358533783000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:25Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 864, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:26Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 864, "optime" : { "t" : 1358533824000, "i" : 3512 }, "optimeDate" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 864, "optime" : { "t" : 1358533825000, "i" : 1049 }, "optimeDate" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:25Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 864, "optime" : { "t" : 1358533783000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 898, "optime" : { "t" : 1358533786000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:29:46Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 864, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:25Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 864, "optime" : { "t" : 1358533825000, "i" : 843 }, "optimeDate" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:26Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 865, "optime" : { "t" : 1358533783000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 30 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 865, "optime" : { "t" : 1358533783000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 25 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 897, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f99424:19cf", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 865, "optime" : { "t" : 1358533824000, "i" : 1247 }, "optimeDate" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 864, "optime" : { "t" : 1358533824000, "i" : 3436 }, "optimeDate" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:25Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 864, "optime" : { "t" : 1358533783000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:24Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 864, "optime" : { "t" : 1358533783000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-18T18:29:43Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:25Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 864, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 897, "optime" : { "t" : 1358533826000, "i" : 3652 }, "optimeDate" : ISODate("2013-01-18T18:30:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } } m31004| Fri Jan 18 13:30:26.704 [conn113] end connection 127.0.0.5:45425 (4 connections now open) m31004| Fri Jan 18 13:30:27.204 [initandlisten] connection accepted from 127.0.0.5:49407 #117 (5 connections now open) m31000| Fri Jan 18 13:30:28.331 [rsHealthPoll] couldn't connect to 127.0.0.5:31003: couldn't connect to server 127.0.0.5:31003 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 904, "optime" : { "t" : 1358533831000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:30:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 870, "optime" : { "t" : 1358533789000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 870, "optime" : { "t" : 1358533789000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532932000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:15:32Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 870, "optime" : { "t" : 1358533831000, "i" : 3780 }, "optimeDate" : ISODate("2013-01-18T18:30:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 869, "optime" : { "t" : 1358533830000, "i" : 3542 }, "optimeDate" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 904, "optime" : { "t" : 1358533789000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 869, "optime" : { "t" : 1358533789000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:31Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 869, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 869, "optime" : { "t" : 1358533830000, "i" : 3542 }, "optimeDate" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 869, "optime" : { "t" : 1358533831000, "i" : 1104 }, "optimeDate" : ISODate("2013-01-18T18:30:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 869, "optime" : { "t" : 1358533789000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 903, "optime" : { "t" : 1358533789000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "self" : true }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 869, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 869, "optime" : { "t" : 1358533831000, "i" : 1087 }, "optimeDate" : ISODate("2013-01-18T18:30:31Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.5:31003 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:31Z"), "myState" : 3, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358532999000, "i" : 1 }, "optimeDate" : ISODate("2013-01-18T18:16:39Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 870, "optime" : { "t" : 1358533789000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:30Z"), "pingMs" : 25 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 870, "optime" : { "t" : 1358533789000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:31Z"), "pingMs" : 30 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 902, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "errmsg" : "still syncing, not yet to minValid optime 50f99424:19cf", "self" : true }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 870, "optime" : { "t" : 1358533829000, "i" : 2797 }, "optimeDate" : ISODate("2013-01-18T18:30:29Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.6:31004 { "set" : "testReplSet", "date" : ISODate("2013-01-18T18:30:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 869, "optime" : { "t" : 1358533830000, "i" : 3225 }, "optimeDate" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 869, "optime" : { "t" : 1358533789000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:30Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 869, "optime" : { "t" : 1358533789000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-18T18:29:49Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-18T18:30:31Z"), "pingMs" : 0 }, { "_id" : 3, "name" : "127.0.0.5:31003", "health" : 1, "state" : 3, "stateStr" : "RECOVERING", "uptime" : 869, "optime" : { "t" : 1358533015000, "i" : 5300 }, "optimeDate" : ISODate("2013-01-18T18:16:55Z"), "lastHeartbeat" : ISODate("2013-01-18T18:30:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "still syncing, not yet to minValid optime 50f99424:19cf" }, { "_id" : 4, "name" : "127.0.0.6:31004", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 902, "optime" : { "t" : 1358533831000, "i" : 6165 }, "optimeDate" : ISODate("2013-01-18T18:30:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.5:31003" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002", "127.0.0.6:31004" ] } }